From 619a7686599e14edb45c7f0763c7e11a3ec6ef69 Mon Sep 17 00:00:00 2001 From: chenheng Date: Wed, 4 Nov 2015 23:54:24 +0800 Subject: [PATCH] HBASE-14756 Break out ClientBackoffPolicy factors into configurable and stackable components --- .../hbase/client/backoff/ClientBackoffPolicy.java | 9 ++++ .../backoff/ExponentialClientBackoffPolicy.java | 19 ++++--- .../hbase/client/TestClientExponentialBackoff.java | 63 ++++++++++++++++++++++ 3 files changed, 85 insertions(+), 6 deletions(-) diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/backoff/ClientBackoffPolicy.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/backoff/ClientBackoffPolicy.java index 94e434f..caab39e 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/backoff/ClientBackoffPolicy.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/backoff/ClientBackoffPolicy.java @@ -35,6 +35,15 @@ public interface ClientBackoffPolicy { public static final String BACKOFF_POLICY_CLASS = "hbase.client.statistics.backoff-policy"; + public static final String BACKOFF_FACTORS_MEMLOAD = + "hbase.client.statistics.backoff.factors.memload"; + + public static final String BACKOFF_FACTORS_HEAP = + "hbase.client.statistics.backoff.factors.heap"; + + public static final String BACKOFF_FACTORS_COMPACTION = + "hbase.client.statistics.backoff.factors.compaction"; + /** * @return the number of ms to wait on the client based on the */ diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/backoff/ExponentialClientBackoffPolicy.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/backoff/ExponentialClientBackoffPolicy.java index b41133a..747beb3 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/backoff/ExponentialClientBackoffPolicy.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/backoff/ExponentialClientBackoffPolicy.java @@ -43,13 +43,19 @@ public class ExponentialClientBackoffPolicy implements ClientBackoffPolicy { private long maxBackoff; private float heapOccupancyLowWatermark; private float heapOccupancyHighWatermark; + private float memoryLoadFactor; + private float heapOccupancyFactor; + private float compactionFactor; public ExponentialClientBackoffPolicy(Configuration conf) { this.maxBackoff = conf.getLong(MAX_BACKOFF_KEY, DEFAULT_MAX_BACKOFF); this.heapOccupancyLowWatermark = conf.getFloat(HConstants.HEAP_OCCUPANCY_LOW_WATERMARK_KEY, - HConstants.DEFAULT_HEAP_OCCUPANCY_LOW_WATERMARK); + HConstants.DEFAULT_HEAP_OCCUPANCY_LOW_WATERMARK); this.heapOccupancyHighWatermark = conf.getFloat(HConstants.HEAP_OCCUPANCY_HIGH_WATERMARK_KEY, - HConstants.DEFAULT_HEAP_OCCUPANCY_HIGH_WATERMARK); + HConstants.DEFAULT_HEAP_OCCUPANCY_HIGH_WATERMARK); + this.memoryLoadFactor = conf.getFloat(BACKOFF_FACTORS_MEMLOAD, 1.0F); + this.heapOccupancyFactor = conf.getFloat(BACKOFF_FACTORS_HEAP, 1.0F); + this.compactionFactor = conf.getFloat(BACKOFF_FACTORS_COMPACTION, 1.0F); } @Override @@ -66,13 +72,14 @@ public class ExponentialClientBackoffPolicy implements ClientBackoffPolicy { } // Factor in memstore load - double percent = regionStats.getMemstoreLoadPercent() / 100.0; - + double percent = this.memoryLoadFactor * regionStats.getMemstoreLoadPercent() / 100.0; // Factor in heap occupancy - float heapOccupancy = regionStats.getHeapOccupancyPercent() / 100.0f; + float heapOccupancy = this.heapOccupancyFactor * + regionStats.getHeapOccupancyPercent() / 100.0f; // Factor in compaction pressure, 1.0 means heavy compaction pressure - float compactionPressure = regionStats.getCompactionPressure() / 100.0f; + float compactionPressure = this.compactionFactor * + regionStats.getCompactionPressure() / 100.0f; if (heapOccupancy >= heapOccupancyLowWatermark) { // If we are higher than the high watermark, we are already applying max // backoff and cannot scale more (see scale() below) diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestClientExponentialBackoff.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestClientExponentialBackoff.java index dcd4fca..be00f81 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestClientExponentialBackoff.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestClientExponentialBackoff.java @@ -19,6 +19,7 @@ package org.apache.hadoop.hbase.client; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.client.backoff.ClientBackoffPolicy; import org.apache.hadoop.hbase.client.backoff.ExponentialClientBackoffPolicy; import org.apache.hadoop.hbase.client.backoff.ServerStatistics; import org.apache.hadoop.hbase.protobuf.generated.ClientProtos; @@ -153,6 +154,68 @@ public class TestClientExponentialBackoff { ExponentialClientBackoffPolicy.DEFAULT_MAX_BACKOFF); } + @Test + public void testFactorConfiguration() { + Configuration conf = new Configuration(false); + conf.setFloat(ClientBackoffPolicy.BACKOFF_FACTORS_MEMLOAD, 0.5F); + conf.setFloat(ClientBackoffPolicy.BACKOFF_FACTORS_COMPACTION, 0F); + conf.setFloat(ClientBackoffPolicy.BACKOFF_FACTORS_HEAP, 0F); + ExponentialClientBackoffPolicy backoff = new ExponentialClientBackoffPolicy(conf); + + ServerStatistics stats = new ServerStatistics(); + long backoffTime; + + update(stats, 100, 0, 0); + backoffTime = backoff.getBackoffTime(server, regionname, stats); + assertEquals( + Math.pow(0.5, 4.0) * ExponentialClientBackoffPolicy.DEFAULT_MAX_BACKOFF, + backoffTime, 0.01); + + conf.setFloat(ClientBackoffPolicy.BACKOFF_FACTORS_MEMLOAD, 0F); + conf.setFloat(ClientBackoffPolicy.BACKOFF_FACTORS_COMPACTION, 0.5F); + conf.setFloat(ClientBackoffPolicy.BACKOFF_FACTORS_HEAP, 0F); + backoff = new ExponentialClientBackoffPolicy(conf); + + update(stats, 0, 50, 0); + backoffTime = backoff.getBackoffTime(server, regionname, stats); + assertEquals(0, backoffTime, 0.01); + update(stats, 0, 96, 0); + assertEquals(0, backoffTime, 0.01); + update(stats, 0, 100, 0); + assertEquals(0, backoffTime, 0.01); + + conf.setFloat(ClientBackoffPolicy.BACKOFF_FACTORS_MEMLOAD, 0F); + conf.setFloat(ClientBackoffPolicy.BACKOFF_FACTORS_COMPACTION, 0F); + conf.setFloat(ClientBackoffPolicy.BACKOFF_FACTORS_HEAP, 0.98F); + backoff = new ExponentialClientBackoffPolicy(conf); + + update(stats, 0, 100, 0); + backoffTime = backoff.getBackoffTime(server, regionname, stats); + assertEquals(ExponentialClientBackoffPolicy.DEFAULT_MAX_BACKOFF, + backoffTime, 0.01); + + conf.setFloat(ClientBackoffPolicy.BACKOFF_FACTORS_MEMLOAD, 0F); + conf.setFloat(ClientBackoffPolicy.BACKOFF_FACTORS_COMPACTION, 0.5F); + conf.setFloat(ClientBackoffPolicy.BACKOFF_FACTORS_HEAP, 0F); + backoff = new ExponentialClientBackoffPolicy(conf); + update(stats, 0, 0, 100); + backoffTime = backoff.getBackoffTime(server, regionname, stats); + assertEquals( + Math.pow(0.5, 4.0) * ExponentialClientBackoffPolicy.DEFAULT_MAX_BACKOFF, + backoffTime, 0.01); + + + conf.setFloat(ClientBackoffPolicy.BACKOFF_FACTORS_MEMLOAD, 0.3F); + conf.setFloat(ClientBackoffPolicy.BACKOFF_FACTORS_COMPACTION, 0.5F); + conf.setFloat(ClientBackoffPolicy.BACKOFF_FACTORS_HEAP, 0.9F); + backoff = new ExponentialClientBackoffPolicy(conf); + update(stats, 100, 100, 100); + backoffTime = backoff.getBackoffTime(server, regionname, stats); + assertEquals( + Math.pow(0.5, 4.0) * ExponentialClientBackoffPolicy.DEFAULT_MAX_BACKOFF, + backoffTime, 0.01); + } + private void update(ServerStatistics stats, int load) { ClientProtos.RegionLoadStats stat = ClientProtos.RegionLoadStats.newBuilder() .setMemstoreLoad -- 2.3.8 (Apple Git-58)