diff --git hbase-common/src/main/resources/hbase-default.xml hbase-common/src/main/resources/hbase-default.xml
index 78e597c..1bfe1b7 100644
--- hbase-common/src/main/resources/hbase-default.xml
+++ hbase-common/src/main/resources/hbase-default.xml
@@ -992,4 +992,32 @@
The default value is 60000ms(60s).
+
+ hbase.regionserver.checksum.verify
+ true
+
+ If set to true, hbase will read data and then verify checksums for
+ hfile blocks. Checksum verification inside hdfs will be switched off.
+ If the hbase-checksum verification fails, then it will switch back to
+ using hdfs checksums.
+
+
+
+ hbase.hstore.bytes.per.checksum
+ 16384
+
+ Number of bytes in a newly created checksum chunk for hbase level
+ checksums in hfile blocks.
+
+
+
+ hbase.hstore.checksum.algorithm
+ CRC32C
+
+ Name of an algorithm that is used to compute checksums. Possible values
+ are NULL, CRC32, CRC32C
+
+
+
+
diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java
index 9b8f3bd..134387f 100644
--- hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java
+++ hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java
@@ -61,7 +61,6 @@ import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos;
import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.BytesBytesPair;
import org.apache.hadoop.hbase.protobuf.generated.HFileProtos;
-import org.apache.hadoop.hbase.regionserver.StoreFile.WriterBuilder;
import org.apache.hadoop.hbase.util.BloomFilterWriter;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.ChecksumType;
@@ -177,7 +176,7 @@ public class HFile {
* The number of bytes per checksum.
*/
public static final int DEFAULT_BYTES_PER_CHECKSUM = 16 * 1024;
- public static final ChecksumType DEFAULT_CHECKSUM_TYPE = ChecksumType.CRC32;
+ public static final ChecksumType DEFAULT_CHECKSUM_TYPE = ChecksumType.CRC32C;
// For measuring latency of "sequential" reads and writes
private static final AtomicInteger readOps = new AtomicInteger();
@@ -193,18 +192,18 @@ public class HFile {
static final AtomicLong checksumFailures = new AtomicLong();
// For getting more detailed stats on FS latencies
- // If, for some reason, the metrics subsystem stops polling for latencies,
+ // If, for some reason, the metrics subsystem stops polling for latencies,
// I don't want data to pile up in a memory leak
// so, after LATENCY_BUFFER_SIZE items have been enqueued for processing,
// fs latency stats will be dropped (and this behavior will be logged)
private static final int LATENCY_BUFFER_SIZE = 5000;
- private static final BlockingQueue fsReadLatenciesNanos =
+ private static final BlockingQueue fsReadLatenciesNanos =
new ArrayBlockingQueue(LATENCY_BUFFER_SIZE);
- private static final BlockingQueue fsWriteLatenciesNanos =
+ private static final BlockingQueue fsWriteLatenciesNanos =
new ArrayBlockingQueue(LATENCY_BUFFER_SIZE);
- private static final BlockingQueue fsPreadLatenciesNanos =
+ private static final BlockingQueue fsPreadLatenciesNanos =
new ArrayBlockingQueue(LATENCY_BUFFER_SIZE);
-
+
public static final void offerReadLatency(long latencyNanos, boolean pread) {
if (pread) {
fsPreadLatenciesNanos.offer(latencyNanos); // might be silently dropped, if the queue is full
@@ -216,30 +215,30 @@ public class HFile {
readOps.incrementAndGet();
}
}
-
+
public static final void offerWriteLatency(long latencyNanos) {
fsWriteLatenciesNanos.offer(latencyNanos); // might be silently dropped, if the queue is full
-
+
writeTimeNano.addAndGet(latencyNanos);
writeOps.incrementAndGet();
}
-
+
public static final Collection getReadLatenciesNanos() {
- final List latencies =
+ final List latencies =
Lists.newArrayListWithCapacity(fsReadLatenciesNanos.size());
fsReadLatenciesNanos.drainTo(latencies);
return latencies;
}
public static final Collection getPreadLatenciesNanos() {
- final List latencies =
+ final List latencies =
Lists.newArrayListWithCapacity(fsPreadLatenciesNanos.size());
fsPreadLatenciesNanos.drainTo(latencies);
return latencies;
}
-
+
public static final Collection getWriteLatenciesNanos() {
- final List latencies =
+ final List latencies =
Lists.newArrayListWithCapacity(fsWriteLatenciesNanos.size());
fsWriteLatenciesNanos.drainTo(latencies);
return latencies;
@@ -589,7 +588,7 @@ public class HFile {
HFileSystem hfs = null;
FSDataInputStream fsdis = fs.open(path);
FSDataInputStream fsdisNoFsChecksum = fsdis;
- // If the fs is not an instance of HFileSystem, then create an
+ // If the fs is not an instance of HFileSystem, then create an
// instance of HFileSystem that wraps over the specified fs.
// In this case, we will not be able to avoid checksumming inside
// the filesystem.
@@ -839,7 +838,7 @@ public class HFile {
/** Now parse the old Writable format. It was a list of Map entries. Each map entry was a key and a value of
* a byte []. The old map format had a byte before each entry that held a code which was short for the key or
* value type. We know it was a byte [] so in below we just read and dump it.
- * @throws IOException
+ * @throws IOException
*/
void parseWritable(final DataInputStream in) throws IOException {
// First clear the map. Otherwise we will just accumulate entries every time this method is called.
diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index 543d30e..2d34de2 100644
--- hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -463,7 +463,16 @@ public class HRegionServer implements ClientProtocol,
// do we use checksum verification in the hbase? If hbase checksum verification
// is enabled, then we automatically switch off hdfs checksum verification.
this.useHBaseChecksum = conf.getBoolean(
- HConstants.HBASE_CHECKSUM_VERIFICATION, false);
+ HConstants.HBASE_CHECKSUM_VERIFICATION, true);
+
+ //check that the user has not set the "dfs.client.read.shortcircuit.skip.checksum" property.
+ boolean shortCircuitSkipChecksum = conf.getBoolean(
+ "dfs.client.read.shortcircuit.skip.checksum", false);
+ if (shortCircuitSkipChecksum) {
+ LOG.warn("Configuration \"dfs.client.read.shortcircuit.skip.checksum\" should not " +
+ "be set to true. See https://issues.apache.org/jira/browse/HBASE-6868");
+ assert !shortCircuitSkipChecksum; //this will fail if assertions are on
+ }
// Config'ed params
this.numRetries = conf.getInt("hbase.client.retries.number", 10);
diff --git src/main/docbkx/performance.xml src/main/docbkx/performance.xml
index 353f7dd..fec4bf5 100644
--- src/main/docbkx/performance.xml
+++ src/main/docbkx/performance.xml
@@ -202,7 +202,11 @@
hbase.regionserver.checksum.verifyHave HBase write the checksum into the datablock and save
- having to do the checksum seek whenever you read. See the
+ having to do the checksum seek whenever you read.
+
+ See ,
+ and
+ For more information see the
release note on HBASE-5074 support checksums in HBase block cache.
@@ -670,7 +674,10 @@ set dfs.client.read.shortcircuit to be true
For optimal performance when short-circuit reads are enabled, it is recommended that HDFS checksums are disabled.
To maintain data integrity with HDFS checksums disabled, HBase can be configured to write its own checksums into
- its datablocks and verify against these. See .
+ its datablocks and verify against these. See . When both
+ local short-circuit reads and hbase level checksums are enabled, you SHOULD NOT disable configuration parameter
+ "dfs.client.read.shortcircuit.skip.checksum", which will cause skipping checksum on non-hfile reads. HBase already
+ manage that setting under the covers.
The DataNodes need to be restarted in order to pick up the new