diff --git a/conf/hbase-env.sh b/conf/hbase-env.sh index 91aca76..e6d1c9c 100644 --- a/conf/hbase-env.sh +++ b/conf/hbase-env.sh @@ -66,10 +66,12 @@ export HBASE_OPTS="-XX:+UseConcMarkSweepGC" # If FILE-PATH is not replaced, the log file(.gc) would still be generated in the HBASE_LOG_DIR . # export CLIENT_GC_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc: -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=1 -XX:GCLogFileSize=512M" -# Uncomment below if you intend to use the EXPERIMENTAL off heap cache. -# export HBASE_OPTS="$HBASE_OPTS -XX:MaxDirectMemorySize=" -# Set hbase.offheapcache.percentage in hbase-site.xml to a nonzero value. - +# Uncomment below if you intend to use off heap cache. +# export HBASE_OPTS="$HBASE_OPTS -XX:MaxDirectMemorySize=SET_THIS_TO_HOW_MANY_GIGS_OF_OFFHEAP" +# For example, to allocate 8G of offheap, set SET_THIS_TO_HOW_MANY_GIGS_OF_OFFHEAP to 8G as in: +# export HBASE_OPTS="$HBASE_OPTS -XX:MaxDirectMemorySize=8G" +# See the package documentation for org.apache.hadoop.hbase.io.hfile for other configurations +# needed setting up off-heap block caching. # Uncomment and adjust to enable JMX exporting # See jmxremote.password and jmxremote.access in $JRE_HOME/lib/management to configure remote password access. diff --git a/hbase-common/src/main/resources/hbase-default.xml b/hbase-common/src/main/resources/hbase-default.xml index f0fafd1..6888b03 100644 --- a/hbase-common/src/main/resources/hbase-default.xml +++ b/hbase-common/src/main/resources/hbase-default.xml @@ -939,8 +939,10 @@ possible configurations would overwhelm and obscure the important. hbase.offheapcache.percentage 0 The percentage of the off heap space (-XX:MaxDirectMemorySize) to be - allocated towards the experimental off heap cache. If you desire the cache to be - disabled, simply set this value to 0. + allocated towards the experimental off heap "SlabCache" (This is different to + the BucketCache -- see the package javadoc for org.apache.hadoop.hbase.io.hfile + for more on your options). If you desire the cache to be disabled, simply set this + value to 0. hbase.data.umask.enable diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java index 59fd2c0..aaabeec 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java @@ -83,7 +83,7 @@ public class CacheConfig { * to the file that will host the file-based cache. See BucketCache#getIOEngineFromName() for * list of supported ioengine options. * - *

Set this option and a non-zero {@link BUCKET_CACHE_SIZE_KEY} to enable bucket cache. + *

Set this option and a non-zero {@link #BUCKET_CACHE_SIZE_KEY} to enable bucket cache. */ public static final String BUCKET_CACHE_IOENGINE_KEY = "hbase.bucketcache.ioengine"; @@ -91,10 +91,10 @@ public class CacheConfig { * When using bucket cache, this is a float that EITHER represents a percentage of total heap * memory size to give to the cache (if < 1.0) OR, it is the capacity in megabytes of the cache. * - *

The resultant size is further divided if {@link BUCKET_CACHE_COMBINED_KEY} is set (It is + *

The resultant size is further divided if {@link #BUCKET_CACHE_COMBINED_KEY} is set (It is * set by default. When false, bucket cache serves as an "L2" cache to the "L1" * {@link LruBlockCache}). The percentage is set in - * with {@link BUCKET_CACHE_COMBINED_PERCENTAGE_KEY} float. + * with {@link #BUCKET_CACHE_COMBINED_PERCENTAGE_KEY} float. */ public static final String BUCKET_CACHE_SIZE_KEY = "hbase.bucketcache.size"; @@ -115,7 +115,7 @@ public class CacheConfig { /** * A float which designates how much of the overall cache to give to bucket cache - * and how much to on-heap lru cache when {@link BUCKET_CACHE_COMBINED_KEY} is set. + * and how much to on-heap lru cache when {@link #BUCKET_CACHE_COMBINED_KEY} is set. */ public static final String BUCKET_CACHE_COMBINED_PERCENTAGE_KEY = "hbase.bucketcache.percentage.in.combinedcache"; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CombinedBlockCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CombinedBlockCache.java index 37221df..a7eabf8 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CombinedBlockCache.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CombinedBlockCache.java @@ -31,7 +31,7 @@ import org.apache.hadoop.hbase.io.hfile.bucket.BucketCache; * CombinedBlockCache is an abstraction layer that combines * {@link LruBlockCache} and {@link BucketCache}. The smaller lruCache is used * to cache bloom blocks and index blocks. The larger bucketCache is used to - * cache data blocks. {@link #getBlock(BlockCacheKey, boolean, boolean) reads + * cache data blocks. {@link #getBlock(BlockCacheKey, boolean, boolean)}, boolean, boolean) reads * first from the smaller lruCache before looking for the block in the bucketCache. * Metrics are the combined size and hits and misses of both caches. * diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/package-info.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/package-info.java index 1f44967..7665186 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/package-info.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/package-info.java @@ -16,7 +16,7 @@ * limitations under the License. */ /** - * Provides implementations of {@link HFile} and HFile + * Provides implementations of {@link org.apache.hadoop.hbase.io.hfile.HFile} and HFile * {@link org.apache.hadoop.hbase.io.hfile.BlockCache}. Caches are configured (and instantiated) * by {@link org.apache.hadoop.hbase.io.hfile.CacheConfig}. See head of the * {@link org.apache.hadoop.hbase.io.hfile.CacheConfig} class for constants that define @@ -32,12 +32,47 @@ * *

Enabling {@link org.apache.hadoop.hbase.io.hfile.slab.SlabCache}

* {@link org.apache.hadoop.hbase.io.hfile.slab.SlabCache} has seen little use and will likely - * be deprecated in the near future. To enable it, - * set the float hbase.offheapcache.percentage to some value between 0 and 1. This + * be deprecated in the near future. It is the original offheap block cache. It is originally + * described in Caching + * in Apache HBase: SlabCache. To enable it, + * set the float hbase.offheapcache.percentage to some value between 0 and 1 in + * your hbase-site.xml file. This * enables {@link org.apache.hadoop.hbase.io.hfile.DoubleBlockCache}, a facade over * {@link org.apache.hadoop.hbase.io.hfile.LruBlockCache} and - * {@link org.apache.hadoop.hbase.io.hfile.slab.SlabCache}. The value set here will be - * multiplied by whatever the setting for -XX:MaxDirectMemorySize is and this is what + * {@link org.apache.hadoop.hbase.io.hfile.slab.SlabCache}. DoubleBlockCache works as follows. + * When caching, it + * "...attempts to cache the block in both caches, while readblock reads first from the faster on heap + * cache before looking for the block in the off heap cache. Metrics are the + * combined size and hits and misses of both caches." The value set in hbase.offheapcache.percentage + * will be + * multiplied by whatever the setting for -XX:MaxDirectMemorySize is in + * your hbase-env.sh configuration file and this is what * will be used by {@link org.apache.hadoop.hbase.io.hfile.slab.SlabCache} as its offheap store. + * Restart your cluster. + * + *

Enabling {@link org.apache.hadoop.hbase.io.hfile.bucket.BucketCache}

+ * Ensure the SlabCache config hbase.offheapcache.percentage is not set (or set to 0). + * At this point, it is probably best to read the code to learn the list of bucket cache options + * and how they combine (to be fixed). Here is a simple example of how to enable a 5G + * offheap bucket cache. The onheap/offheap caches + * are managed by {@link org.apache.hadoop.hbase.io.hfile.CombinedBlockCache}. For the + * CombinedBlockCache, "The smaller lruCache is used + * to cache bloom blocks and index blocks, the larger bucketCache is used to + * cache data blocks. getBlock reads first from the smaller lruCache before + * looking for the block in the bucketCache. Metrics are the combined size and + * hits and misses of both caches." + *

In + * hbase-env.sh ensure the java option -XX:MaxDirectMemorySize is + * enabled and at least 5G in size: e.g. -XX:MaxDirectMemorySize=5G. Then in + * hbase-site.xml add the following configurations: +

<property>
+  <name>hbase.bucketcache.ioengine</name>
+  <value>offheap</value>
+</property>
+<property>
+  <name>hbase.bucketcache.size</name>
+  <value>5120</value>
+</property>
and then restart. + * */ package org.apache.hadoop.hbase.io.hfile; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java index 20110f2..ca6f444 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java @@ -337,8 +337,7 @@ public interface HLog { * able to sync an explicit edit only (the current default implementation syncs up to the time * of the sync call syncing whatever is behind the sync). * @throws IOException - * @deprecated Use - * {@link #appendNoSync(HRegionInfo, HLogKey, WALEdit, HTableDescriptor, AtomicLong, boolean)} + * @deprecated Use {@link #appendNoSync(HTableDescriptor, HRegionInfo, HLogKey, WALEdit, AtomicLong, boolean)} * instead because you can get back the region edit/sequenceid; it is set into the passed in * key. */ diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java index b772223..9e92d5e 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java @@ -131,7 +131,7 @@ import com.google.protobuf.Service; * *

* To perform authorization checks, {@code AccessController} relies on the - * {@link org.apache.hadoop.hbase.ipc.RpcServerEngine} being loaded to provide + * RpcServerEngine being loaded to provide * the user identities for remote requests. *

*