Index: hbase-client/src/test/java/org/apache/hadoop/hbase/ipc/TestPayloadCarryingRpcController.java =================================================================== --- hbase-client/src/test/java/org/apache/hadoop/hbase/ipc/TestPayloadCarryingRpcController.java (revision 1515273) +++ hbase-client/src/test/java/org/apache/hadoop/hbase/ipc/TestPayloadCarryingRpcController.java (working copy) @@ -165,7 +165,7 @@ } @Override - public int getTagsLength() { + public short getTagsLength() { // TODO Auto-generated method stub return 0; } Index: hbase-common/src/main/java/org/apache/hadoop/hbase/Cell.java =================================================================== --- hbase-common/src/main/java/org/apache/hadoop/hbase/Cell.java (revision 1515273) +++ hbase-common/src/main/java/org/apache/hadoop/hbase/Cell.java (working copy) @@ -181,6 +181,6 @@ /** * @return the total length of the tags in the Cell. */ - int getTagsLength(); + short getTagsLength(); } Index: hbase-common/src/main/java/org/apache/hadoop/hbase/KeyValue.java =================================================================== --- hbase-common/src/main/java/org/apache/hadoop/hbase/KeyValue.java (revision 1515273) +++ hbase-common/src/main/java/org/apache/hadoop/hbase/KeyValue.java (working copy) @@ -2824,7 +2824,7 @@ } @Override - public int getTagsLength() { + public short getTagsLength() { throw new UnsupportedOperationException("Not implememnted"); } Index: hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/decode/PrefixTreeCell.java =================================================================== --- hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/decode/PrefixTreeCell.java (revision 1515273) +++ hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/decode/PrefixTreeCell.java (working copy) @@ -200,7 +200,7 @@ } @Override - public int getTagsLength() { + public short getTagsLength() { throw new UnsupportedOperationException("Not implemented"); } Index: hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/PrefixTreeBlockMeta.java =================================================================== --- hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/PrefixTreeBlockMeta.java (revision 1515273) +++ hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/PrefixTreeBlockMeta.java (working copy) @@ -46,7 +46,7 @@ public static final int NUM_LONGS = 2, - NUM_INTS = 22, + NUM_INTS = 28, NUM_SHORTS = 0,//keyValueTypeWidth not persisted NUM_SINGLE_BYTES = 2, MAX_BYTES = Bytes.SIZEOF_LONG * NUM_LONGS @@ -76,6 +76,7 @@ protected int numTimestampBytes; protected int numMvccVersionBytes; protected int numValueBytes; + protected int numTagBytes; // number of bytes in each section of fixed width FInts protected int nextNodeOffsetWidth; @@ -85,11 +86,13 @@ protected int mvccVersionIndexWidth; protected int valueOffsetWidth; protected int valueLengthWidth; + protected int tagOffsetWidth; // used to pre-allocate structures for reading protected int rowTreeDepth; protected int maxRowLength; protected int maxQualifierLength; + protected int maxTagLength; // the timestamp from which the deltas are calculated protected long minTimestamp; @@ -103,6 +106,7 @@ protected int numUniqueRows; protected int numUniqueFamilies; protected int numUniqueQualifiers; + protected int numUniqueTags; /***************** constructors ********************/ @@ -143,6 +147,7 @@ numBytes += UVIntTool.numBytes(numRowBytes); numBytes += UVIntTool.numBytes(numFamilyBytes); numBytes += UVIntTool.numBytes(numQualifierBytes); + numBytes += UVIntTool.numBytes(numTagBytes); numBytes += UVIntTool.numBytes(numTimestampBytes); numBytes += UVIntTool.numBytes(numMvccVersionBytes); numBytes += UVIntTool.numBytes(numValueBytes); @@ -150,6 +155,7 @@ numBytes += UVIntTool.numBytes(nextNodeOffsetWidth); numBytes += UVIntTool.numBytes(familyOffsetWidth); numBytes += UVIntTool.numBytes(qualifierOffsetWidth); + numBytes += UVIntTool.numBytes(tagOffsetWidth); numBytes += UVIntTool.numBytes(timestampIndexWidth); numBytes += UVIntTool.numBytes(mvccVersionIndexWidth); numBytes += UVIntTool.numBytes(valueOffsetWidth); @@ -158,6 +164,7 @@ numBytes += UVIntTool.numBytes(rowTreeDepth); numBytes += UVIntTool.numBytes(maxRowLength); numBytes += UVIntTool.numBytes(maxQualifierLength); + numBytes += UVIntTool.numBytes(maxTagLength); numBytes += UVLongTool.numBytes(minTimestamp); numBytes += UVIntTool.numBytes(timestampDeltaWidth); @@ -169,6 +176,7 @@ numBytes += UVIntTool.numBytes(numUniqueRows); numBytes += UVIntTool.numBytes(numUniqueFamilies); numBytes += UVIntTool.numBytes(numUniqueQualifiers); + numBytes += UVIntTool.numBytes(numUniqueTags); return numBytes; } @@ -181,6 +189,7 @@ UVIntTool.writeBytes(numRowBytes, os); UVIntTool.writeBytes(numFamilyBytes, os); UVIntTool.writeBytes(numQualifierBytes, os); + UVIntTool.writeBytes(numTagBytes, os); UVIntTool.writeBytes(numTimestampBytes, os); UVIntTool.writeBytes(numMvccVersionBytes, os); UVIntTool.writeBytes(numValueBytes, os); @@ -188,6 +197,7 @@ UVIntTool.writeBytes(nextNodeOffsetWidth, os); UVIntTool.writeBytes(familyOffsetWidth, os); UVIntTool.writeBytes(qualifierOffsetWidth, os); + UVIntTool.writeBytes(tagOffsetWidth, os); UVIntTool.writeBytes(timestampIndexWidth, os); UVIntTool.writeBytes(mvccVersionIndexWidth, os); UVIntTool.writeBytes(valueOffsetWidth, os); @@ -196,6 +206,7 @@ UVIntTool.writeBytes(rowTreeDepth, os); UVIntTool.writeBytes(maxRowLength, os); UVIntTool.writeBytes(maxQualifierLength, os); + UVIntTool.writeBytes(maxTagLength, os); UVLongTool.writeBytes(minTimestamp, os); UVIntTool.writeBytes(timestampDeltaWidth, os); @@ -207,6 +218,7 @@ UVIntTool.writeBytes(numUniqueRows, os); UVIntTool.writeBytes(numUniqueFamilies, os); UVIntTool.writeBytes(numUniqueQualifiers, os); + UVIntTool.writeBytes(numUniqueTags, os); } public void readVariableBytesFromInputStream(InputStream is) throws IOException{ @@ -218,6 +230,7 @@ numRowBytes = UVIntTool.getInt(is); numFamilyBytes = UVIntTool.getInt(is); numQualifierBytes = UVIntTool.getInt(is); + numTagBytes = UVIntTool.getInt(is); numTimestampBytes = UVIntTool.getInt(is); numMvccVersionBytes = UVIntTool.getInt(is); numValueBytes = UVIntTool.getInt(is); @@ -225,6 +238,7 @@ nextNodeOffsetWidth = UVIntTool.getInt(is); familyOffsetWidth = UVIntTool.getInt(is); qualifierOffsetWidth = UVIntTool.getInt(is); + tagOffsetWidth = UVIntTool.getInt(is); timestampIndexWidth = UVIntTool.getInt(is); mvccVersionIndexWidth = UVIntTool.getInt(is); valueOffsetWidth = UVIntTool.getInt(is); @@ -233,6 +247,7 @@ rowTreeDepth = UVIntTool.getInt(is); maxRowLength = UVIntTool.getInt(is); maxQualifierLength = UVIntTool.getInt(is); + maxTagLength = UVIntTool.getInt(is); minTimestamp = UVLongTool.getLong(is); timestampDeltaWidth = UVIntTool.getInt(is); @@ -245,6 +260,7 @@ numUniqueRows = UVIntTool.getInt(is); numUniqueFamilies = UVIntTool.getInt(is); numUniqueQualifiers = UVIntTool.getInt(is); + numUniqueTags = UVIntTool.getInt(is); } public void readVariableBytesFromArray(byte[] bytes, int offset) { @@ -265,6 +281,8 @@ position += UVIntTool.numBytes(numFamilyBytes); numQualifierBytes = UVIntTool.getInt(bytes, position); position += UVIntTool.numBytes(numQualifierBytes); + numTagBytes = UVIntTool.getInt(bytes, position); + position += UVIntTool.numBytes(numTagBytes); numTimestampBytes = UVIntTool.getInt(bytes, position); position += UVIntTool.numBytes(numTimestampBytes); numMvccVersionBytes = UVIntTool.getInt(bytes, position); @@ -278,6 +296,8 @@ position += UVIntTool.numBytes(familyOffsetWidth); qualifierOffsetWidth = UVIntTool.getInt(bytes, position); position += UVIntTool.numBytes(qualifierOffsetWidth); + tagOffsetWidth = UVIntTool.getInt(bytes, position); + position += UVIntTool.numBytes(tagOffsetWidth); timestampIndexWidth = UVIntTool.getInt(bytes, position); position += UVIntTool.numBytes(timestampIndexWidth); mvccVersionIndexWidth = UVIntTool.getInt(bytes, position); @@ -293,7 +313,8 @@ position += UVIntTool.numBytes(maxRowLength); maxQualifierLength = UVIntTool.getInt(bytes, position); position += UVIntTool.numBytes(maxQualifierLength); - + maxTagLength = UVIntTool.getInt(bytes, position); + position += UVIntTool.numBytes(maxTagLength); minTimestamp = UVLongTool.getLong(bytes, position); position += UVLongTool.numBytes(minTimestamp); timestampDeltaWidth = UVIntTool.getInt(bytes, position); @@ -314,6 +335,8 @@ position += UVIntTool.numBytes(numUniqueFamilies); numUniqueQualifiers = UVIntTool.getInt(bytes, position); position += UVIntTool.numBytes(numUniqueQualifiers); + numUniqueTags = UVIntTool.getInt(bytes, position); + position += UVIntTool.numBytes(numUniqueTags); } //TODO method that can read directly from ByteBuffer instead of InputStream @@ -396,6 +419,8 @@ return false; if (maxQualifierLength != other.maxQualifierLength) return false; + if (maxTagLength != other.maxTagLength) + return false; if (maxRowLength != other.maxRowLength) return false; if (mvccVersionDeltaWidth != other.mvccVersionDeltaWidth) @@ -418,6 +443,8 @@ return false; if (numQualifierBytes != other.numQualifierBytes) return false; + if (numTagBytes != other.numTagBytes) + return false; if (numRowBytes != other.numRowBytes) return false; if (numTimestampBytes != other.numTimestampBytes) @@ -426,12 +453,16 @@ return false; if (numUniqueQualifiers != other.numUniqueQualifiers) return false; + if (numUniqueTags != other.numUniqueTags) + return false; if (numUniqueRows != other.numUniqueRows) return false; if (numKeyValueBytes != other.numKeyValueBytes) return false; if (qualifierOffsetWidth != other.qualifierOffsetWidth) return false; + if(tagOffsetWidth != other.tagOffsetWidth) + return false; if (rowTreeDepth != other.rowTreeDepth) return false; if (timestampDeltaWidth != other.timestampDeltaWidth) @@ -459,6 +490,7 @@ result = prime * result + familyOffsetWidth; result = prime * result + (includesMvccVersion ? 1231 : 1237); result = prime * result + maxQualifierLength; + result = prime * result + maxTagLength; result = prime * result + maxRowLength; result = prime * result + mvccVersionDeltaWidth; result = prime * result + mvccVersionIndexWidth; @@ -470,13 +502,16 @@ result = prime * result + numMvccVersionBytes; result = prime * result + numMetaBytes; result = prime * result + numQualifierBytes; + result = prime * result + numTagBytes; result = prime * result + numRowBytes; result = prime * result + numTimestampBytes; result = prime * result + numUniqueFamilies; result = prime * result + numUniqueQualifiers; + result = prime * result + numUniqueTags; result = prime * result + numUniqueRows; result = prime * result + numKeyValueBytes; result = prime * result + qualifierOffsetWidth; + result = prime * result + tagOffsetWidth; result = prime * result + rowTreeDepth; result = prime * result + timestampDeltaWidth; result = prime * result + timestampIndexWidth; @@ -514,12 +549,16 @@ builder.append(numMvccVersionBytes); builder.append(", numValueBytes="); builder.append(numValueBytes); + builder.append(", numTagBytes="); + builder.append(numTagBytes); builder.append(", nextNodeOffsetWidth="); builder.append(nextNodeOffsetWidth); builder.append(", familyOffsetWidth="); builder.append(familyOffsetWidth); builder.append(", qualifierOffsetWidth="); builder.append(qualifierOffsetWidth); + builder.append(", tagOffsetWidth="); + builder.append(tagOffsetWidth); builder.append(", timestampIndexWidth="); builder.append(timestampIndexWidth); builder.append(", mvccVersionIndexWidth="); @@ -534,6 +573,8 @@ builder.append(maxRowLength); builder.append(", maxQualifierLength="); builder.append(maxQualifierLength); + builder.append(", maxTagLength="); + builder.append(maxTagLength); builder.append(", minTimestamp="); builder.append(minTimestamp); builder.append(", timestampDeltaWidth="); @@ -552,6 +593,8 @@ builder.append(numUniqueFamilies); builder.append(", numUniqueQualifiers="); builder.append(numUniqueQualifiers); + builder.append(", numUniqueTags="); + builder.append(numUniqueTags); builder.append("]"); return builder.toString(); } @@ -575,10 +618,14 @@ return getAbsoluteFamilyOffset() + numFamilyBytes; } - public int getAbsoluteTimestampOffset() { + public int getAbsoluteTagOffset() { return getAbsoluteQualifierOffset() + numQualifierBytes; } + public int getAbsoluteTimestampOffset() { + return getAbsoluteTagOffset() + numTagBytes; + } + public int getAbsoluteMvccVersionOffset() { return getAbsoluteTimestampOffset() + numTimestampBytes; } @@ -602,10 +649,18 @@ return valueOffsetWidth; } + public int getTagOffsetWidth() { + return tagOffsetWidth; + } + public void setValueOffsetWidth(int dataOffsetWidth) { this.valueOffsetWidth = dataOffsetWidth; } + public void setTagOffsetWidth(int dataOffsetWidth) { + this.tagOffsetWidth = dataOffsetWidth; + } + public int getValueLengthWidth() { return valueLengthWidth; } @@ -674,6 +729,14 @@ return numValueBytes; } + public int getNumTagBytes() { + return numTagBytes; + } + + public void setNumTagBytes(int numTagBytes){ + this.numTagBytes = numTagBytes; + } + public void setNumValueBytes(int numValueBytes) { this.numValueBytes = numValueBytes; } @@ -782,6 +845,13 @@ this.numUniqueQualifiers = numUniqueQualifiers; } + public void setNumUniqueTags(int numUniqueTags) { + this.numUniqueTags = numUniqueTags; + } + + public int getNumUniqueTags() { + return numUniqueTags; + } public int getNumQualifierBytes() { return numQualifierBytes; } @@ -802,10 +872,19 @@ return maxQualifierLength; } + // TODO : decide on some max value for this ? INTEGER_MAX? public void setMaxQualifierLength(int maxQualifierLength) { this.maxQualifierLength = maxQualifierLength; } + public int getMaxTagLength() { + return this.maxTagLength; + } + + public void setMaxTagLength(int maxTagLength) { + this.maxTagLength = maxTagLength; + } + public int getTimestampIndexWidth() { return timestampIndexWidth; } Index: hbase-prefix-tree/src/test/java/org/apache/hadoop/hbase/codec/prefixtree/blockmeta/TestBlockMeta.java =================================================================== --- hbase-prefix-tree/src/test/java/org/apache/hadoop/hbase/codec/prefixtree/blockmeta/TestBlockMeta.java (revision 1515273) +++ hbase-prefix-tree/src/test/java/org/apache/hadoop/hbase/codec/prefixtree/blockmeta/TestBlockMeta.java (working copy) @@ -39,6 +39,7 @@ m.setNumRowBytes(0); m.setNumFamilyBytes(3); m.setNumQualifierBytes(12345); + m.setNumTagBytes(50); m.setNumTimestampBytes(23456); m.setNumMvccVersionBytes(5); m.setNumValueBytes(34567); @@ -46,6 +47,7 @@ m.setNextNodeOffsetWidth(3); m.setFamilyOffsetWidth(1); m.setQualifierOffsetWidth(2); + m.setTagOffsetWidth(2); m.setTimestampIndexWidth(1); m.setMvccVersionIndexWidth(2); m.setValueOffsetWidth(8); @@ -54,6 +56,7 @@ m.setRowTreeDepth(11); m.setMaxRowLength(200); m.setMaxQualifierLength(50); + m.setMaxTagLength(40); m.setMinTimestamp(1318966363481L); m.setTimestampDeltaWidth(3); @@ -66,6 +69,7 @@ m.setNumUniqueRows(88); m.setNumUniqueFamilies(1); m.setNumUniqueQualifiers(56); + m.setNumUniqueTags(5); return m; } Index: hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALCellCodec.java =================================================================== --- hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALCellCodec.java (revision 1515273) +++ hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALCellCodec.java (working copy) @@ -156,6 +156,8 @@ // We first write the KeyValue infrastructure as VInts. StreamUtils.writeRawVInt32(out, kv.getKeyLength()); StreamUtils.writeRawVInt32(out, kv.getValueLength()); + // To support tags + StreamUtils.writeRawVInt32(out, kv.getTagsLength()); // Write row, qualifier, and family; use dictionary // compression as they're likely to have duplicates. @@ -196,7 +198,10 @@ protected Cell parseCell() throws IOException { int keylength = StreamUtils.readRawVarint32(in); int vlength = StreamUtils.readRawVarint32(in); - int length = KeyValue.KEYVALUE_INFRASTRUCTURE_SIZE + keylength + vlength; + + // To support Tags..Tags length will be 0 + int tagsLength = StreamUtils.readRawVarint32(in); + int length = KeyValue.KEYVALUE_INFRASTRUCTURE_SIZE + keylength + vlength + tagsLength; byte[] backingArray = new byte[length]; int pos = 0; @@ -221,7 +226,7 @@ // the rest IOUtils.readFully(in, backingArray, pos, length - pos); - return new KeyValue(backingArray); + return new KeyValue(backingArray, 0, length); } private int readIntoArray(byte[] to, int offset, Dictionary dict) throws IOException {