diff --git hbase-common/src/main/resources/hbase-default.xml hbase-common/src/main/resources/hbase-default.xml index 78e597c..1bfe1b7 100644 --- hbase-common/src/main/resources/hbase-default.xml +++ hbase-common/src/main/resources/hbase-default.xml @@ -992,4 +992,32 @@ The default value is 60000ms(60s). + + hbase.regionserver.checksum.verify + true + + If set to true, hbase will read data and then verify checksums for + hfile blocks. Checksum verification inside hdfs will be switched off. + If the hbase-checksum verification fails, then it will switch back to + using hdfs checksums. + + + + hbase.hstore.bytes.per.checksum + 16384 + + Number of bytes in a newly created checksum chunk for hbase level + checksums in hfile blocks. + + + + hbase.hstore.checksum.algorithm + CRC32C + + Name of an algorithm that is used to compute checksums. Possible values + are NULL, CRC32, CRC32C + + + + diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java index 9b8f3bd..134387f 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java @@ -61,7 +61,6 @@ import org.apache.hadoop.hbase.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos; import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.BytesBytesPair; import org.apache.hadoop.hbase.protobuf.generated.HFileProtos; -import org.apache.hadoop.hbase.regionserver.StoreFile.WriterBuilder; import org.apache.hadoop.hbase.util.BloomFilterWriter; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.ChecksumType; @@ -177,7 +176,7 @@ public class HFile { * The number of bytes per checksum. */ public static final int DEFAULT_BYTES_PER_CHECKSUM = 16 * 1024; - public static final ChecksumType DEFAULT_CHECKSUM_TYPE = ChecksumType.CRC32; + public static final ChecksumType DEFAULT_CHECKSUM_TYPE = ChecksumType.CRC32C; // For measuring latency of "sequential" reads and writes private static final AtomicInteger readOps = new AtomicInteger(); @@ -193,18 +192,18 @@ public class HFile { static final AtomicLong checksumFailures = new AtomicLong(); // For getting more detailed stats on FS latencies - // If, for some reason, the metrics subsystem stops polling for latencies, + // If, for some reason, the metrics subsystem stops polling for latencies, // I don't want data to pile up in a memory leak // so, after LATENCY_BUFFER_SIZE items have been enqueued for processing, // fs latency stats will be dropped (and this behavior will be logged) private static final int LATENCY_BUFFER_SIZE = 5000; - private static final BlockingQueue fsReadLatenciesNanos = + private static final BlockingQueue fsReadLatenciesNanos = new ArrayBlockingQueue(LATENCY_BUFFER_SIZE); - private static final BlockingQueue fsWriteLatenciesNanos = + private static final BlockingQueue fsWriteLatenciesNanos = new ArrayBlockingQueue(LATENCY_BUFFER_SIZE); - private static final BlockingQueue fsPreadLatenciesNanos = + private static final BlockingQueue fsPreadLatenciesNanos = new ArrayBlockingQueue(LATENCY_BUFFER_SIZE); - + public static final void offerReadLatency(long latencyNanos, boolean pread) { if (pread) { fsPreadLatenciesNanos.offer(latencyNanos); // might be silently dropped, if the queue is full @@ -216,30 +215,30 @@ public class HFile { readOps.incrementAndGet(); } } - + public static final void offerWriteLatency(long latencyNanos) { fsWriteLatenciesNanos.offer(latencyNanos); // might be silently dropped, if the queue is full - + writeTimeNano.addAndGet(latencyNanos); writeOps.incrementAndGet(); } - + public static final Collection getReadLatenciesNanos() { - final List latencies = + final List latencies = Lists.newArrayListWithCapacity(fsReadLatenciesNanos.size()); fsReadLatenciesNanos.drainTo(latencies); return latencies; } public static final Collection getPreadLatenciesNanos() { - final List latencies = + final List latencies = Lists.newArrayListWithCapacity(fsPreadLatenciesNanos.size()); fsPreadLatenciesNanos.drainTo(latencies); return latencies; } - + public static final Collection getWriteLatenciesNanos() { - final List latencies = + final List latencies = Lists.newArrayListWithCapacity(fsWriteLatenciesNanos.size()); fsWriteLatenciesNanos.drainTo(latencies); return latencies; @@ -589,7 +588,7 @@ public class HFile { HFileSystem hfs = null; FSDataInputStream fsdis = fs.open(path); FSDataInputStream fsdisNoFsChecksum = fsdis; - // If the fs is not an instance of HFileSystem, then create an + // If the fs is not an instance of HFileSystem, then create an // instance of HFileSystem that wraps over the specified fs. // In this case, we will not be able to avoid checksumming inside // the filesystem. @@ -839,7 +838,7 @@ public class HFile { /** Now parse the old Writable format. It was a list of Map entries. Each map entry was a key and a value of * a byte []. The old map format had a byte before each entry that held a code which was short for the key or * value type. We know it was a byte [] so in below we just read and dump it. - * @throws IOException + * @throws IOException */ void parseWritable(final DataInputStream in) throws IOException { // First clear the map. Otherwise we will just accumulate entries every time this method is called. diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java index 543d30e..2d34de2 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java @@ -463,7 +463,16 @@ public class HRegionServer implements ClientProtocol, // do we use checksum verification in the hbase? If hbase checksum verification // is enabled, then we automatically switch off hdfs checksum verification. this.useHBaseChecksum = conf.getBoolean( - HConstants.HBASE_CHECKSUM_VERIFICATION, false); + HConstants.HBASE_CHECKSUM_VERIFICATION, true); + + //check that the user has not set the "dfs.client.read.shortcircuit.skip.checksum" property. + boolean shortCircuitSkipChecksum = conf.getBoolean( + "dfs.client.read.shortcircuit.skip.checksum", false); + if (shortCircuitSkipChecksum) { + LOG.warn("Configuration \"dfs.client.read.shortcircuit.skip.checksum\" should not " + + "be set to true. See https://issues.apache.org/jira/browse/HBASE-6868"); + assert !shortCircuitSkipChecksum; //this will fail if assertions are on + } // Config'ed params this.numRetries = conf.getInt("hbase.client.retries.number", 10); diff --git src/main/docbkx/performance.xml src/main/docbkx/performance.xml index 353f7dd..fec4bf5 100644 --- src/main/docbkx/performance.xml +++ src/main/docbkx/performance.xml @@ -202,7 +202,11 @@
<varname>hbase.regionserver.checksum.verify</varname> Have HBase write the checksum into the datablock and save - having to do the checksum seek whenever you read. See the + having to do the checksum seek whenever you read. + + See , + and + For more information see the release note on HBASE-5074 support checksums in HBase block cache.
@@ -670,7 +674,10 @@ set dfs.client.read.shortcircuit to be true For optimal performance when short-circuit reads are enabled, it is recommended that HDFS checksums are disabled. To maintain data integrity with HDFS checksums disabled, HBase can be configured to write its own checksums into - its datablocks and verify against these. See . + its datablocks and verify against these. See . When both + local short-circuit reads and hbase level checksums are enabled, you SHOULD NOT disable configuration parameter + "dfs.client.read.shortcircuit.skip.checksum", which will cause skipping checksum on non-hfile reads. HBase already + manage that setting under the covers. The DataNodes need to be restarted in order to pick up the new