Index: src/test/java/org/apache/hadoop/hbase/regionserver/DataBlockEncodingTool.java =================================================================== --- src/test/java/org/apache/hadoop/hbase/regionserver/DataBlockEncodingTool.java (revision 1310596) +++ src/test/java/org/apache/hadoop/hbase/regionserver/DataBlockEncodingTool.java (working copy) @@ -119,6 +119,7 @@ private long totalCFLength = 0; private byte[] rawKVs; + private int minorVersion = 0; private final String compressionAlgorithmName; private final Algorithm compressionAlgorithm; @@ -228,7 +229,7 @@ List> codecIterators = new ArrayList>(); for(EncodedDataBlock codec : codecs) { - codecIterators.add(codec.getIterator()); + codecIterators.add(codec.getIterator(minorVersion)); } int j = 0; @@ -320,7 +321,7 @@ Iterator it; - it = codec.getIterator(); + it = codec.getIterator(minorVersion); // count only the algorithm time, without memory allocations // (expect first time) @@ -592,6 +593,7 @@ // run the utilities DataBlockEncodingTool comp = new DataBlockEncodingTool(compressionName); + comp.minorVersion = reader.getHFileMinorVersion(); comp.checkStatistics(scanner, kvLimit); if (doVerify) { comp.verifyCodecs(scanner, kvLimit); Index: src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlock.java =================================================================== --- src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlock.java (revision 1310596) +++ src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlock.java (working copy) @@ -446,7 +446,7 @@ new byte[rawBuf.array().length + headerLen]; System.arraycopy(rawBuf.array(), 0, rawBufWithHeader, headerLen, rawBuf.array().length); - defaultEncodingCtx.compressAfterEncoding(rawBufWithHeader, + defaultEncodingCtx.compressAfterEncodingWithBlockType(rawBufWithHeader, BlockType.DATA); encodedResultWithHeader = defaultEncodingCtx.getUncompressedBytesWithHeader(); Index: src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockCompatibility.java =================================================================== --- src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockCompatibility.java (revision 1310596) +++ src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockCompatibility.java (working copy) @@ -530,7 +530,7 @@ if (blockType == BlockType.DATA) { encodeDataBlockForDisk(); } else { - defaultBlockEncodingCtx.compressAfterEncoding( + defaultBlockEncodingCtx.compressAfterEncodingWithBlockType( uncompressedBytesWithHeader, blockType); onDiskBytesWithHeader = defaultBlockEncodingCtx.getOnDiskBytesWithHeader(); Index: src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileDataBlockEncoder.java =================================================================== --- src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileDataBlockEncoder.java (revision 1310596) +++ src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileDataBlockEncoder.java (working copy) @@ -127,6 +127,7 @@ HFileBlockEncodingContext context = new HFileBlockDefaultEncodingContext( Compression.Algorithm.NONE, blockEncoder.getEncodingOnDisk()); + context.setDummyHeader(block.getDummyHeaderForVersion()); blockEncoder.beforeWriteToDisk(block.getBufferWithoutHeader(), includesMemstoreTS, context, block.getBlockType()); Index: src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java (revision 1310596) +++ src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java (working copy) @@ -1698,6 +1698,10 @@ return reader.getTrailer().getMajorVersion(); } + public int getHFileMinorVersion() { + return reader.getTrailer().getMinorVersion(); + } + HFile.Reader getHFileReader() { return reader; } Index: src/main/java/org/apache/hadoop/hbase/io/encoding/HFileBlockDefaultEncodingContext.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/io/encoding/HFileBlockDefaultEncodingContext.java (revision 1310596) +++ src/main/java/org/apache/hadoop/hbase/io/encoding/HFileBlockDefaultEncodingContext.java (working copy) @@ -59,7 +59,7 @@ private ByteArrayOutputStream encodedStream = new ByteArrayOutputStream(); private DataOutputStream dataOut = new DataOutputStream(encodedStream); - private final byte[] dummyHeader; + private byte[] dummyHeader; /** * @param compressionAlgorithm compression algorithm used @@ -92,6 +92,12 @@ } } + @Override + public void setDummyHeader(byte[] headerBytes) + { + dummyHeader = headerBytes; + } + /** * @param compressionAlgorithm compression algorithm * @param encoding encoding @@ -119,7 +125,7 @@ public void postEncoding(BlockType blockType) throws IOException { dataOut.flush(); - compressAfterEncoding(encodedStream.toByteArray(), blockType); + compressAfterEncodingWithBlockType(encodedStream.toByteArray(), blockType); this.blockType = blockType; } @@ -128,11 +134,23 @@ * @param blockType * @throws IOException */ - public void compressAfterEncoding(byte[] uncompressedBytesWithHeader, + public void compressAfterEncodingWithBlockType(byte[] uncompressedBytesWithHeader, BlockType blockType) throws IOException { - compressAfterEncoding(uncompressedBytesWithHeader, blockType, dummyHeader); + compressAfterEncoding(uncompressedBytesWithHeader, blockType, + dummyHeader); } + /* + * @param uncompressedBytesWithHeader + * @param block HFileBlock + * @throws IOException + */ + public void compressAfterEncoding(byte[] uncompressedBytesWithHeader, + HFileBlock block) throws IOException { + compressAfterEncoding(uncompressedBytesWithHeader, block.getBlockType(), + block.getDummyHeaderForVersion()); + } + /** * @param uncompressedBytesWithHeader * @param blockType @@ -199,10 +217,4 @@ public DataBlockEncoding getDataBlockEncoding() { return this.encodingAlgo; } - - @Override - public int getHeaderSize() { - return this.dummyHeader.length; - } - } Index: src/main/java/org/apache/hadoop/hbase/io/encoding/EncodedDataBlock.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/io/encoding/EncodedDataBlock.java (revision 1310596) +++ src/main/java/org/apache/hadoop/hbase/io/encoding/EncodedDataBlock.java (working copy) @@ -71,12 +71,13 @@ /** * Provides access to compressed value. + * @param minorVersion minor version of HFileBlock * @return Forwards sequential iterator. */ - public Iterator getIterator() { + public Iterator getIterator(int minorVersion) { final int rawSize = rawKVs.length; byte[] encodedDataWithHeader = getEncodedData(); - int bytesToSkip = encodingCtx.getHeaderSize() + Bytes.SIZEOF_SHORT; + int bytesToSkip = HFileBlock.headerSize(minorVersion) + Bytes.SIZEOF_SHORT; ByteArrayInputStream bais = new ByteArrayInputStream(encodedDataWithHeader, bytesToSkip, encodedDataWithHeader.length - bytesToSkip); final DataInputStream dis = new DataInputStream(bais); Index: src/main/java/org/apache/hadoop/hbase/io/encoding/HFileBlockEncodingContext.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/io/encoding/HFileBlockEncodingContext.java (revision 1310596) +++ src/main/java/org/apache/hadoop/hbase/io/encoding/HFileBlockEncodingContext.java (working copy) @@ -53,9 +53,9 @@ public Compression.Algorithm getCompression(); /** - * @return the header size used + * sets the dummy header bytes */ - public int getHeaderSize(); + public void setDummyHeader(byte[] headerBytes); /** * @return the {@link DataBlockEncoding} encoding used Index: src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java (revision 1310596) +++ src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java (working copy) @@ -821,7 +821,7 @@ if (blockType == BlockType.DATA) { encodeDataBlockForDisk(); } else { - defaultBlockEncodingCtx.compressAfterEncoding( + defaultBlockEncodingCtx.compressAfterEncodingWithBlockType( uncompressedBytesWithHeader, blockType); onDiskBytesWithHeader = defaultBlockEncodingCtx.getOnDiskBytesWithHeader(); @@ -1903,7 +1903,7 @@ /** * Maps a minor version to the size of the header. */ - static private int headerSize(int minorVersion) { + static public int headerSize(int minorVersion) { if (minorVersion < MINOR_VERSION_WITH_CHECKSUM) { return HEADER_SIZE_NO_CHECKSUM; } @@ -1911,6 +1911,23 @@ } /** + * Return the appropriate DUMMY_HEADER for the minor version + */ + public byte[] getDummyHeaderForVersion() { + return getDummyHeaderForVersion(minorVersion); + } + + /** + * Return the appropriate DUMMY_HEADER for the minor version + */ + static private byte[] getDummyHeaderForVersion(int minorVersion) { + if (minorVersion < MINOR_VERSION_WITH_CHECKSUM) { + return DUMMY_HEADER_NO_CHECKSUM; + } + return DUMMY_HEADER; + } + + /** * Convert the contents of the block header into a human readable string. * This is mostly helpful for debugging. This assumes that the block * has minor version > 0. Index: src/main/java/org/apache/hadoop/hbase/io/hfile/HFileDataBlockEncoderImpl.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/io/hfile/HFileDataBlockEncoderImpl.java (revision 1310596) +++ src/main/java/org/apache/hadoop/hbase/io/hfile/HFileDataBlockEncoderImpl.java (working copy) @@ -71,7 +71,6 @@ */ public HFileDataBlockEncoderImpl(DataBlockEncoding onDisk, DataBlockEncoding inCache, byte[] dummyHeader) { - dummyHeader = dummyHeader == null ? HFileBlock.DUMMY_HEADER : dummyHeader; this.onDisk = onDisk != null ? onDisk : DataBlockEncoding.NONE; this.inCache = inCache != null ? @@ -96,19 +95,26 @@ public static HFileDataBlockEncoder createFromFileInfo( FileInfo fileInfo, DataBlockEncoding preferredEncodingInCache) throws IOException { - byte[] dataBlockEncodingType = - fileInfo.get(StoreFile.DATA_BLOCK_ENCODING); - if (dataBlockEncodingType == null) { + + boolean hasPreferredCacheEncoding = preferredEncodingInCache != null + && preferredEncodingInCache != DataBlockEncoding.NONE; + + byte[] dataBlockEncodingType = fileInfo.get(StoreFile.DATA_BLOCK_ENCODING); + if (dataBlockEncodingType == null && !hasPreferredCacheEncoding) { return NoOpDataBlockEncoder.INSTANCE; } - String dataBlockEncodingStr = Bytes.toString(dataBlockEncodingType); DataBlockEncoding onDisk; - try { - onDisk = DataBlockEncoding.valueOf(dataBlockEncodingStr); - } catch (IllegalArgumentException ex) { - throw new IOException("Invalid data block encoding type in file info: " + - dataBlockEncodingStr, ex); + if (dataBlockEncodingType == null) { + onDisk = DataBlockEncoding.NONE; + } else { + String dataBlockEncodingStr = Bytes.toString(dataBlockEncodingType); + try { + onDisk = DataBlockEncoding.valueOf(dataBlockEncodingStr); + } catch (IllegalArgumentException ex) { + throw new IOException("Invalid data block encoding type in file info: " + + dataBlockEncodingStr, ex); + } } DataBlockEncoding inCache; @@ -192,7 +198,7 @@ BlockType blockType) throws IOException { if (onDisk == DataBlockEncoding.NONE) { // there is no need to encode the block before writing it to disk - ((HFileBlockDefaultEncodingContext) encodeCtx).compressAfterEncoding( + ((HFileBlockDefaultEncodingContext) encodeCtx).compressAfterEncodingWithBlockType( in.array(), blockType); return; } @@ -234,12 +240,13 @@ private HFileBlock encodeDataBlock(HFileBlock block, DataBlockEncoding algo, boolean includesMemstoreTS, HFileBlockEncodingContext encodingCtx) { + inCacheEncodeCtx.setDummyHeader(block.getDummyHeaderForVersion()); encodeBufferToHFileBlockBuffer( block.getBufferWithoutHeader(), algo, includesMemstoreTS, encodingCtx); byte[] encodedUncompressedBytes = encodingCtx.getUncompressedBytesWithHeader(); ByteBuffer bufferWrapper = ByteBuffer.wrap(encodedUncompressedBytes); - int sizeWithoutHeader = bufferWrapper.limit() - encodingCtx.getHeaderSize(); + int sizeWithoutHeader = bufferWrapper.limit() - block.headerSize(); HFileBlock encodedBlock = new HFileBlock(BlockType.ENCODED_DATA, block.getOnDiskSizeWithoutHeader(), sizeWithoutHeader, block.getPrevBlockOffset(), Index: src/main/java/org/apache/hadoop/hbase/io/hfile/NoOpDataBlockEncoder.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/io/hfile/NoOpDataBlockEncoder.java (revision 1310596) +++ src/main/java/org/apache/hadoop/hbase/io/hfile/NoOpDataBlockEncoder.java (working copy) @@ -62,7 +62,7 @@ HFileBlockDefaultEncodingContext defaultContext = (HFileBlockDefaultEncodingContext) encodeCtx; - defaultContext.compressAfterEncoding(in.array(), blockType); + defaultContext.compressAfterEncodingWithBlockType(in.array(), blockType); } @Override Index: src/main/java/org/apache/hadoop/hbase/io/hfile/FixedFileTrailer.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/io/hfile/FixedFileTrailer.java (revision 1310596) +++ src/main/java/org/apache/hadoop/hbase/io/hfile/FixedFileTrailer.java (working copy) @@ -465,7 +465,7 @@ /** * Returns the minor version of this HFile format */ - int getMinorVersion() { + public int getMinorVersion() { return minorVersion; }