diff --git a/hbase-common/src/main/resources/hbase-default.xml b/hbase-common/src/main/resources/hbase-default.xml
index fa5d522..938359e 100644
--- a/hbase-common/src/main/resources/hbase-default.xml
+++ b/hbase-common/src/main/resources/hbase-default.xml
@@ -615,13 +615,14 @@ possible configurations would overwhelm and obscure the important.
hbase.hregion.percolumnfamilyflush.size.lower.bound
16777216
- If FlushLargeStoresPolicy is used, then every time that we hit the
- total memstore limit, we find out all the column families whose memstores
- exceed this value, and only flush them, while retaining the others whose
- memstores are lower than this limit. If none of the families have their
- memstore size more than this, all the memstores will be flushed
- (just as usual). This value should be less than half of the total memstore
- threshold (hbase.hregion.memstore.flush.size).
+ If FlushLargeStoresPolicy is used and there are multiple column families,
+ then every time that we hit the total memstore limit, we find out all the
+ column families whose memstores exceed a "lower bound" and only flush them
+ while retaining the others in memory. The "lower bound" will be
+ "hbase.hregion.memstore.flush.size / column_family_number" by default
+ unless value of this property is larger than that. If none of the families
+ have their memstore size more than lower bound, all the memstores will be
+ flushed (just as usual).
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/FlushLargeStoresPolicy.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/FlushLargeStoresPolicy.java
index 328e890..6ba49aa 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/FlushLargeStoresPolicy.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/FlushLargeStoresPolicy.java
@@ -40,33 +40,43 @@ public class FlushLargeStoresPolicy extends FlushPolicy {
private static final long DEFAULT_HREGION_COLUMNFAMILY_FLUSH_SIZE_LOWER_BOUND = 1024 * 1024 * 16L;
- private long flushSizeLowerBound;
+ private long flushSizeLowerBound = -1;
@Override
protected void configureForRegion(HRegion region) {
super.configureForRegion(region);
- long flushSizeLowerBound;
+ int familyNumber = region.getTableDesc().getFamilies().size();
+ if (familyNumber == 1) {
+ // no need to parse and set flush size lower bound if only one family
+ return;
+ }
+ // For multiple families, lower bound is the "average flush size" by default
+ // unless setting in configuration is larger.
+ long flushSizeLowerBound = region.getMemstoreFlushSize() / familyNumber;
+ long lowerBoundFromConf =
+ getConf().getLong(HREGION_COLUMNFAMILY_FLUSH_SIZE_LOWER_BOUND,
+ DEFAULT_HREGION_COLUMNFAMILY_FLUSH_SIZE_LOWER_BOUND);
+ if (lowerBoundFromConf > flushSizeLowerBound) {
+ flushSizeLowerBound = lowerBoundFromConf;
+ }
+ // use the setting in table description if any
String flushedSizeLowerBoundString =
region.getTableDesc().getValue(HREGION_COLUMNFAMILY_FLUSH_SIZE_LOWER_BOUND);
if (flushedSizeLowerBoundString == null) {
- flushSizeLowerBound =
- getConf().getLong(HREGION_COLUMNFAMILY_FLUSH_SIZE_LOWER_BOUND,
- DEFAULT_HREGION_COLUMNFAMILY_FLUSH_SIZE_LOWER_BOUND);
if (LOG.isDebugEnabled()) {
- LOG.debug(HREGION_COLUMNFAMILY_FLUSH_SIZE_LOWER_BOUND
- + " is not specified, use global config(" + flushSizeLowerBound + ") instead");
+ LOG.debug("No " + HREGION_COLUMNFAMILY_FLUSH_SIZE_LOWER_BOUND
+ + " set in description of table " + region.getTableDesc().getTableName()
+ + ", use config (" + flushSizeLowerBound + ") instead");
}
} else {
try {
flushSizeLowerBound = Long.parseLong(flushedSizeLowerBoundString);
} catch (NumberFormatException nfe) {
- flushSizeLowerBound =
- getConf().getLong(HREGION_COLUMNFAMILY_FLUSH_SIZE_LOWER_BOUND,
- DEFAULT_HREGION_COLUMNFAMILY_FLUSH_SIZE_LOWER_BOUND);
+ // fall back for fault setting
LOG.warn("Number format exception when parsing "
+ HREGION_COLUMNFAMILY_FLUSH_SIZE_LOWER_BOUND + " for table "
+ region.getTableDesc().getTableName() + ":" + flushedSizeLowerBoundString + ". " + nfe
- + ", use global config(" + flushSizeLowerBound + ") instead");
+ + ", use config (" + flushSizeLowerBound + ") instead");
}
}
@@ -87,6 +97,11 @@ public class FlushLargeStoresPolicy extends FlushPolicy {
@Override
public Collection selectStoresToFlush() {
+ // no need to select stores if only one family
+ if (region.getTableDesc().getFamilies().size() == 1) {
+ return region.stores.values();
+ }
+ // start selection
Collection stores = region.stores.values();
Set specificStoresToFlush = new HashSet();
for (Store store : stores) {
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index 7bf4855..e5c63fb 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -8091,4 +8091,8 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
return this.getRegionInfo().isMetaRegion() ? CellComparator.META_COMPARATOR
: CellComparator.COMPARATOR;
}
+
+ public long getMemstoreFlushSize() {
+ return this.memstoreFlushSize;
+ }
}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestPerColumnFamilyFlush.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestPerColumnFamilyFlush.java
index 0df2799..e0c92ff 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestPerColumnFamilyFlush.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestPerColumnFamilyFlush.java
@@ -568,7 +568,6 @@ public class TestPerColumnFamilyFlush {
Configuration conf = TEST_UTIL.getConfiguration();
conf.setLong(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, memstoreFlushSize);
conf.set(FlushPolicyFactory.HBASE_FLUSH_POLICY_KEY, FlushAllStoresPolicy.class.getName());
- conf.setLong(FlushLargeStoresPolicy.HREGION_COLUMNFAMILY_FLUSH_SIZE_LOWER_BOUND, 400 * 1024);
conf.setInt(HStore.BLOCKING_STOREFILES_KEY, 10000);
conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
ConstantSizeRegionSplitPolicy.class.getName());
@@ -608,6 +607,8 @@ public class TestPerColumnFamilyFlush {
LOG.info("==============Test with selective flush enabled===============");
conf.set(FlushPolicyFactory.HBASE_FLUSH_POLICY_KEY, FlushLargeStoresPolicy.class.getName());
+ // default value of per-cf flush lower bound is too big, set to a small enough value
+ conf.setLong(FlushLargeStoresPolicy.HREGION_COLUMNFAMILY_FLUSH_SIZE_LOWER_BOUND, 0);
try {
TEST_UTIL.startMiniCluster(1);
TEST_UTIL.getHBaseAdmin().createNamespace(