diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java index abad835..9cf216b 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java @@ -75,11 +75,7 @@ import org.apache.hadoop.hbase.io.HFileLink; import org.apache.hadoop.hbase.io.HalfStoreFileReader; import org.apache.hadoop.hbase.io.Reference; import org.apache.hadoop.hbase.io.compress.Compression.Algorithm; -import org.apache.hadoop.hbase.io.hfile.CacheConfig; -import org.apache.hadoop.hbase.io.hfile.HFile; -import org.apache.hadoop.hbase.io.hfile.HFileContext; -import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder; -import org.apache.hadoop.hbase.io.hfile.HFileScanner; +import org.apache.hadoop.hbase.io.hfile.*; import org.apache.hadoop.hbase.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.regionserver.BloomType; import org.apache.hadoop.hbase.regionserver.HStore; @@ -854,6 +850,11 @@ public class LoadIncrementalHFiles extends Configured implements Tool { } private static boolean shouldCopyHFileMetaKey(byte[] key) { + // skip encoding to keep hfile meta consistent with data block info, see HBASE-15085 + if (Bytes.equals(key, HFileDataBlockEncoder.DATA_BLOCK_ENCODING)) { + return false; + } + return !HFile.isReservedFileInfoKey(key); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFiles.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFiles.java index 6a8051d..0d857f2 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFiles.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFiles.java @@ -35,6 +35,7 @@ import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding; import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.TableNotFoundException; import org.apache.hadoop.hbase.NamespaceDescriptor; @@ -461,6 +462,44 @@ public class TestLoadIncrementalHFiles { assertEquals(1000, rowCount); } + @Test + public void testSplitStoreFileWithEncodedToNone() throws IOException { + testSplitStoreFileWithDifferentEncoding(DataBlockEncoding.DIFF, DataBlockEncoding.NONE); + } + + @Test + public void testSplitStoreFileWithNoneToEncoded() throws IOException { + testSplitStoreFileWithDifferentEncoding(DataBlockEncoding.NONE, DataBlockEncoding.DIFF); + } + + private void testSplitStoreFileWithDifferentEncoding(DataBlockEncoding bulkloadEncoding, + DataBlockEncoding cfEncoding) throws IOException { + Path dir = util.getDataTestDirOnTestFS("testSplitStoreFileWith" + + bulkloadEncoding + "To" + cfEncoding); + FileSystem fs = util.getTestFileSystem(); + Path testIn = new Path(dir, "testhfile"); + HColumnDescriptor familyDesc = new HColumnDescriptor(FAMILY); + // force DATA_BLOCK_ENCODING to NONE in CF description + familyDesc.setDataBlockEncoding(cfEncoding); + // create HFile with DIFF encoding algorithm + HFileTestUtil.createHFileWithDataBlockEncoding( + util.getConfiguration(), fs, testIn, bulkloadEncoding, + FAMILY, QUALIFIER, Bytes.toBytes("aaa"), Bytes.toBytes("zzz"), 1000); + + Path bottomOut = new Path(dir, "bottom.out"); + Path topOut = new Path(dir, "top.out"); + + LoadIncrementalHFiles.splitStoreFile( + util.getConfiguration(), testIn, + familyDesc, Bytes.toBytes("ggg"), + bottomOut, + topOut); + + int rowCount = verifyHFile(bottomOut); + rowCount += verifyHFile(topOut); + assertEquals(1000, rowCount); + } + private int verifyHFile(Path p) throws IOException { Configuration conf = util.getConfiguration(); HFile.Reader reader = HFile.createReader( diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/HFileTestUtil.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/HFileTestUtil.java index 5d264e1..fb67104 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/HFileTestUtil.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/HFileTestUtil.java @@ -29,6 +29,7 @@ import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.ResultScanner; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding; import org.apache.hadoop.hbase.io.hfile.CacheConfig; import org.apache.hadoop.hbase.io.hfile.HFile; import org.apache.hadoop.hbase.io.hfile.HFileContext; @@ -57,7 +58,21 @@ public class HFileTestUtil { FileSystem fs, Path path, byte[] family, byte[] qualifier, byte[] startKey, byte[] endKey, int numRows) throws IOException { - createHFile(configuration, fs, path, family, qualifier, startKey, endKey, + createHFile(configuration, fs, path, DataBlockEncoding.NONE, family, qualifier, + startKey, endKey, numRows, false); + } + + /** + * Create an HFile with the given number of rows between a given + * start key and end key @ family:qualifier. The value will be the key value. + * This file will use certain data block encoding algorithm. + */ + public static void createHFileWithDataBlockEncoding( + Configuration configuration, + FileSystem fs, Path path, DataBlockEncoding encoding, + byte[] family, byte[] qualifier, + byte[] startKey, byte[] endKey, int numRows) throws IOException { + createHFile(configuration, fs, path, encoding, family, qualifier, startKey, endKey, numRows, false); } @@ -71,7 +86,8 @@ public class HFileTestUtil { FileSystem fs, Path path, byte[] family, byte[] qualifier, byte[] startKey, byte[] endKey, int numRows) throws IOException { - createHFile(configuration, fs, path, family, qualifier, startKey, endKey, numRows, true); + createHFile(configuration, fs, path, DataBlockEncoding.NONE, family, qualifier, + startKey, endKey, numRows, true); } /** @@ -82,11 +98,12 @@ public class HFileTestUtil { */ public static void createHFile( Configuration configuration, - FileSystem fs, Path path, + FileSystem fs, Path path, DataBlockEncoding encoding, byte[] family, byte[] qualifier, byte[] startKey, byte[] endKey, int numRows, boolean withTag) throws IOException { HFileContext meta = new HFileContextBuilder() .withIncludesTags(withTag) + .withDataBlockEncoding(encoding) .build(); HFile.Writer writer = HFile.getWriterFactory(configuration, new CacheConfig(configuration)) .withPath(fs, path)