Index: hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java =================================================================== --- hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java (revision 1509842) +++ hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java (working copy) @@ -500,6 +500,16 @@ // is enabled, then we automatically switch off hdfs checksum verification. this.useHBaseChecksum = conf.getBoolean(HConstants.HBASE_CHECKSUM_VERIFICATION, false); + // check that the user has not set the "dfs.client.read.shortcircuit.skip.checksum" property. + boolean shortCircuitSkipChecksum = conf.getBoolean( + "dfs.client.read.shortcircuit.skip.checksum", false); + if (shortCircuitSkipChecksum) { + LOG.warn("Configuration \"dfs.client.read.shortcircuit.skip.checksum\" should not " + + "be set to true." + (this.useHBaseChecksum ? " HBase checksum doesn't require " + + "it, see https://issues.apache.org/jira/browse/HBASE-6868." : "")); + assert !shortCircuitSkipChecksum; //this will fail if assertions are on + } + // Config'ed params this.numRetries = this.conf.getInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, HConstants.DEFAULT_HBASE_CLIENT_RETRIES_NUMBER); Index: hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java =================================================================== --- hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java (revision 1509842) +++ hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java (working copy) @@ -62,7 +62,6 @@ import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.BytesBytesPair; import org.apache.hadoop.hbase.protobuf.generated.HFileProtos; import org.apache.hadoop.hbase.io.FSDataInputStreamWrapper; -import org.apache.hadoop.hbase.regionserver.StoreFile.WriterBuilder; import org.apache.hadoop.hbase.util.BloomFilterWriter; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.ChecksumType; @@ -178,7 +177,7 @@ * The number of bytes per checksum. */ public static final int DEFAULT_BYTES_PER_CHECKSUM = 16 * 1024; - public static final ChecksumType DEFAULT_CHECKSUM_TYPE = ChecksumType.CRC32; + public static final ChecksumType DEFAULT_CHECKSUM_TYPE = ChecksumType.CRC32C; // For measuring latency of "sequential" reads and writes private static final AtomicInteger readOps = new AtomicInteger(); @@ -194,18 +193,18 @@ static final AtomicLong checksumFailures = new AtomicLong(); // For getting more detailed stats on FS latencies - // If, for some reason, the metrics subsystem stops polling for latencies, + // If, for some reason, the metrics subsystem stops polling for latencies, // I don't want data to pile up in a memory leak // so, after LATENCY_BUFFER_SIZE items have been enqueued for processing, // fs latency stats will be dropped (and this behavior will be logged) private static final int LATENCY_BUFFER_SIZE = 5000; - private static final BlockingQueue fsReadLatenciesNanos = + private static final BlockingQueue fsReadLatenciesNanos = new ArrayBlockingQueue(LATENCY_BUFFER_SIZE); - private static final BlockingQueue fsWriteLatenciesNanos = + private static final BlockingQueue fsWriteLatenciesNanos = new ArrayBlockingQueue(LATENCY_BUFFER_SIZE); - private static final BlockingQueue fsPreadLatenciesNanos = + private static final BlockingQueue fsPreadLatenciesNanos = new ArrayBlockingQueue(LATENCY_BUFFER_SIZE); - + public static final void offerReadLatency(long latencyNanos, boolean pread) { if (pread) { fsPreadLatenciesNanos.offer(latencyNanos); // might be silently dropped, if the queue is full @@ -217,30 +216,30 @@ readOps.incrementAndGet(); } } - + public static final void offerWriteLatency(long latencyNanos) { fsWriteLatenciesNanos.offer(latencyNanos); // might be silently dropped, if the queue is full - + writeTimeNano.addAndGet(latencyNanos); writeOps.incrementAndGet(); } - + public static final Collection getReadLatenciesNanos() { - final List latencies = + final List latencies = Lists.newArrayListWithCapacity(fsReadLatenciesNanos.size()); fsReadLatenciesNanos.drainTo(latencies); return latencies; } public static final Collection getPreadLatenciesNanos() { - final List latencies = + final List latencies = Lists.newArrayListWithCapacity(fsPreadLatenciesNanos.size()); fsPreadLatenciesNanos.drainTo(latencies); return latencies; } - + public static final Collection getWriteLatenciesNanos() { - final List latencies = + final List latencies = Lists.newArrayListWithCapacity(fsWriteLatenciesNanos.size()); fsWriteLatenciesNanos.drainTo(latencies); return latencies; @@ -822,7 +821,7 @@ /** Now parse the old Writable format. It was a list of Map entries. Each map entry was a key and a value of * a byte []. The old map format had a byte before each entry that held a code which was short for the key or * value type. We know it was a byte [] so in below we just read and dump it. - * @throws IOException + * @throws IOException */ void parseWritable(final DataInputStream in) throws IOException { // First clear the map. Otherwise we will just accumulate entries every time this method is called. Index: src/main/docbkx/performance.xml =================================================================== --- src/main/docbkx/performance.xml (revision 1509842) +++ src/main/docbkx/performance.xml (working copy) @@ -202,7 +202,11 @@
<varname>hbase.regionserver.checksum.verify</varname> Have HBase write the checksum into the datablock and save - having to do the checksum seek whenever you read. See the + having to do the checksum seek whenever you read. + + See , + and + For more information see the release note on HBASE-5074 support checksums in HBase block cache.
@@ -670,7 +674,10 @@ For optimal performance when short-circuit reads are enabled, it is recommended that HDFS checksums are disabled. To maintain data integrity with HDFS checksums disabled, HBase can be configured to write its own checksums into - its datablocks and verify against these. See . + its datablocks and verify against these. See . When both + local short-circuit reads and hbase level checksums are enabled, you SHOULD NOT disable configuration parameter + "dfs.client.read.shortcircuit.skip.checksum", which will cause skipping checksum on non-hfile reads. HBase already + manages that setting under the covers. The DataNodes need to be restarted in order to pick up the new Index: hbase-common/src/main/resources/hbase-default.xml =================================================================== --- hbase-common/src/main/resources/hbase-default.xml (revision 1509842) +++ hbase-common/src/main/resources/hbase-default.xml (working copy) @@ -931,4 +931,32 @@ datanode, performing block recovery to timeout on a dead datanode; usually dfs.socket.timeout. See the end of HBASE-8389 for more. + + hbase.regionserver.checksum.verify + true + + If set to true, HBase will read data and then verify checksums for + hfile blocks. Checksum verification inside HDFS will be switched off. + If the hbase-checksum verification fails, then it will switch back to + using HDFS checksums. + + + + hbase.hstore.bytes.per.checksum + 16384 + + Number of bytes in a newly created checksum chunk for HBase-level + checksums in hfile blocks. + + + + hbase.hstore.checksum.algorithm + CRC32C + + Name of an algorithm that is used to compute checksums. Possible values + are NULL, CRC32, CRC32C. + + + +