From d583414393f81eb7a3d1ebcaf8202210963aa9dc Mon Sep 17 00:00:00 2001 From: "hongxi.sy" Date: Wed, 14 Sep 2016 09:53:43 +0800 Subject: [PATCH] to #EHB-446 offheap hfile format should keep compatible with v2/v3 --- .../hadoop/hbase/io/hfile/CompoundBloomFilter.java | 11 ++++++----- .../hbase/io/hfile/CompoundBloomFilterWriter.java | 14 +++++++++++--- .../hadoop/hbase/io/hfile/FixedFileTrailer.java | 20 +++++++++++++------- 3 files changed, 30 insertions(+), 15 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CompoundBloomFilter.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CompoundBloomFilter.java index 2d773bb..69da93d 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CompoundBloomFilter.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CompoundBloomFilter.java @@ -73,11 +73,12 @@ public class CompoundBloomFilter extends CompoundBloomFilterBase totalMaxKeys = meta.readLong(); numChunks = meta.readInt(); byte[] comparatorClassName = Bytes.readByteArray(meta); - // The writer would have return 0 as the vint length for the case of - // Bytes.BYTES_RAWCOMPARATOR. In such cases do not initialize comparator, it can be - // null - if (comparatorClassName.length != 0) { - comparator = FixedFileTrailer.createComparator(Bytes.toString(comparatorClassName)); + String comparatorClassNameString = Bytes.toString(comparatorClassName); + if (comparatorClassNameString.equals(KeyValue.RAW_COMPARATOR.getLegacyKeyComparatorName()) + ||comparatorClassNameString.equals(KeyValue.RAW_COMPARATOR.getClass().getName())) { + comparator = null; + }else { + comparator = FixedFileTrailer.createComparator(comparatorClassNameString); } hash = Hash.getInstance(hashType); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CompoundBloomFilterWriter.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CompoundBloomFilterWriter.java index cd60562..ff95a8e 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CompoundBloomFilterWriter.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CompoundBloomFilterWriter.java @@ -29,6 +29,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.CellComparator; +import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.util.BloomFilterChunk; import org.apache.hadoop.hbase.util.BloomFilterUtil; import org.apache.hadoop.hbase.util.BloomFilterWriter; @@ -244,11 +245,18 @@ public class CompoundBloomFilterWriter extends CompoundBloomFilterBase // Fields that don't have equivalents in ByteBloomFilter. out.writeInt(numChunks); + String comparatorName = null; if (comparator != null) { - Bytes.writeByteArray(out, Bytes.toBytes(comparator.getClass().getName())); + if (!comparator.getClass().equals(CellComparator.COMPARATOR.getClass())) { + throw new IOException("comparator should be of type CellComparator!"); + } + //bloom filter comparator should be compatible with hfile v2 + comparatorName = KeyValue.COMPARATOR.getLegacyKeyComparatorName(); + Bytes.writeByteArray(out, Bytes.toBytes(comparatorName)); } else { - // Internally writes a 0 vint if the byte[] is null - Bytes.writeByteArray(out, null); + //bloom filter comparator should be compatible with hfile v2 + comparatorName = KeyValue.RAW_COMPARATOR.getLegacyKeyComparatorName(); + Bytes.writeByteArray(out, Bytes.toBytes(comparatorName)); } // Write a single-level index without compression or block header. diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/FixedFileTrailer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/FixedFileTrailer.java index 72f550a..09b7fb6 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/FixedFileTrailer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/FixedFileTrailer.java @@ -542,16 +542,22 @@ public class FixedFileTrailer { public void setComparatorClass(Class klass) { // Is the comparator instantiable? try { - // If null, it should be the Bytes.BYTES_RAWCOMPARATOR - if (klass != null) { - CellComparator comp = klass.newInstance(); - // if the name wasn't one of the legacy names, maybe its a legit new - // kind of comparator. + CellComparator comp = klass.newInstance(); + + // HFile V2 legacy comparator class names. + if (CellComparator.COMPARATOR.getClass().equals(klass)) { + comparatorClassName = KeyValue.COMPARATOR.getLegacyKeyComparatorName(); + } else if (CellComparator.META_COMPARATOR.getClass().equals(klass)) { + comparatorClassName = KeyValue.META_COMPARATOR.getLegacyKeyComparatorName(); + } else if (KeyValue.RAW_COMPARATOR.getClass().equals(klass)) { + comparatorClassName = KeyValue.RAW_COMPARATOR.getLegacyKeyComparatorName(); + } else { + // if the name wasn't one of the legacy names, maybe its a legit new kind of comparator. comparatorClassName = klass.getName(); } - } catch (Exception e) { - throw new RuntimeException("Comparator class " + klass.getName() + " is not instantiable", e); + throw new RuntimeException("Comparator class " + klass.getName() + + " is not instantiable", e); } } -- 1.8.3.1