From 48681e1ae46c09a8db99f5b625dfa6ac6031c91f Mon Sep 17 00:00:00 2001 From: Abhishek Singh Chouhan Date: Mon, 17 Jul 2017 21:07:01 +0530 Subject: [PATCH] HBASE-18374 RegionServer Metrics improvements --- .../regionserver/MetricsRegionServerSource.java | 31 ++++++++++++++++++---- .../MetricsRegionServerSourceImpl.java | 26 ++++++++++++++++-- .../regionserver/MetricsRegionSourceImpl.java | 2 +- .../hbase/regionserver/MetricsRegionServer.java | 14 +++++++++- .../hadoop/hbase/regionserver/RSRpcServices.java | 31 +++++++++++++++++++--- .../regionserver/TestMetricsRegionServer.java | 17 +++++++++--- .../regionserver/TestRegionServerMetrics.java | 2 +- .../regionserver/TestRemoveRegionMetrics.java | 4 +-- 8 files changed, 108 insertions(+), 19 deletions(-) diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java index 0e7dd8d..fc2fc45 100644 --- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java @@ -61,6 +61,24 @@ public interface MetricsRegionServerSource extends BaseSource, JvmPauseMonitorSo void updateDelete(long t); /** + * Update the Delete time histogram if a batch contains a delete op + * @param t time it took + */ + void updateDeleteBatch(long t); + + /** + * Update checkAndDelete histogram + * @param t time it took + */ + void updateCheckAndDelete(long t); + + /** + * Update checkAndPut histogram + * @param t time it took + */ + void updateCheckAndPut(long t); + + /** * Update the Get time histogram . * * @param t time it took @@ -334,25 +352,28 @@ public interface MetricsRegionServerSource extends BaseSource, JvmPauseMonitorSo String UPDATES_BLOCKED_DESC = "Number of MS updates have been blocked so that the memstore can be flushed."; String DELETE_KEY = "delete"; + String CHECK_AND_DELETE_KEY = "checkAndDelete"; + String CHECK_AND_PUT_KEY = "checkAndPut"; + String DELETE_BATCH_KEY = "deleteBatch"; String GET_SIZE_KEY = "getSize"; String GET_KEY = "get"; String INCREMENT_KEY = "increment"; - String MUTATE_KEY = "mutate"; + String PUT_KEY = "put"; String APPEND_KEY = "append"; String REPLAY_KEY = "replay"; String SCAN_KEY = "scan"; String SCAN_SIZE_KEY = "scanSize"; String SCAN_TIME_KEY = "scanTime"; - String SLOW_MUTATE_KEY = "slowPutCount"; + String SLOW_PUT_KEY = "slowPutCount"; String SLOW_GET_KEY = "slowGetCount"; String SLOW_DELETE_KEY = "slowDeleteCount"; String SLOW_INCREMENT_KEY = "slowIncrementCount"; String SLOW_APPEND_KEY = "slowAppendCount"; - String SLOW_MUTATE_DESC = - "The number of Multis that took over 1000ms to complete"; + String SLOW_PUT_DESC = + "The number of puts(and batches containing puts) that took over 1000ms to complete"; String SLOW_DELETE_DESC = - "The number of Deletes that took over 1000ms to complete"; + "The number of batches containing delete(s) that took over 1000ms to complete"; String SLOW_GET_DESC = "The number of Gets that took over 1000ms to complete"; String SLOW_INCREMENT_DESC = "The number of Increments that took over 1000ms to complete"; diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java index 67764f8..319147a 100644 --- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java @@ -38,6 +38,9 @@ public class MetricsRegionServerSourceImpl final MetricsRegionServerWrapper rsWrap; private final MetricHistogram putHisto; private final MetricHistogram deleteHisto; + private final MetricHistogram deleteBatchHisto; + private final MetricHistogram checkAndDeleteHisto; + private final MetricHistogram checkAndPutHisto; private final MetricHistogram getHisto; private final MetricHistogram incrementHisto; private final MetricHistogram appendHisto; @@ -97,12 +100,16 @@ public class MetricsRegionServerSourceImpl super(metricsName, metricsDescription, metricsContext, metricsJmxContext); this.rsWrap = rsWrap; - putHisto = getMetricsRegistry().newTimeHistogram(MUTATE_KEY); - slowPut = getMetricsRegistry().newCounter(SLOW_MUTATE_KEY, SLOW_MUTATE_DESC, 0L); + putHisto = getMetricsRegistry().newTimeHistogram(PUT_KEY); + slowPut = getMetricsRegistry().newCounter(SLOW_PUT_KEY, SLOW_PUT_DESC, 0L); deleteHisto = getMetricsRegistry().newTimeHistogram(DELETE_KEY); slowDelete = getMetricsRegistry().newCounter(SLOW_DELETE_KEY, SLOW_DELETE_DESC, 0L); + deleteBatchHisto = getMetricsRegistry().newTimeHistogram(DELETE_BATCH_KEY); + checkAndDeleteHisto = getMetricsRegistry().newTimeHistogram(CHECK_AND_DELETE_KEY); + checkAndPutHisto = getMetricsRegistry().newTimeHistogram(CHECK_AND_PUT_KEY); + getHisto = getMetricsRegistry().newTimeHistogram(GET_KEY); slowGet = getMetricsRegistry().newCounter(SLOW_GET_KEY, SLOW_GET_DESC, 0L); @@ -537,4 +544,19 @@ public class MetricsRegionServerSourceImpl public void updatePauseTimeWithoutGc(long t) { pausesWithoutGc.add(t); } + + @Override + public void updateDeleteBatch(long t) { + deleteBatchHisto.add(t); + } + + @Override + public void updateCheckAndDelete(long t) { + checkAndDeleteHisto.add(t); + } + + @Override + public void updateCheckAndPut(long t) { + checkAndPutHisto.add(t); + } } diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSourceImpl.java index 17710e1..d544a2e 100644 --- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSourceImpl.java +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSourceImpl.java @@ -83,7 +83,7 @@ public class MetricsRegionSourceImpl implements MetricsRegionSource { String suffix = "Count"; - regionPutKey = regionNamePrefix + MetricsRegionServerSource.MUTATE_KEY + suffix; + regionPutKey = regionNamePrefix + MetricsRegionServerSource.PUT_KEY + suffix; regionPut = registry.getCounter(regionPutKey, 0L); regionDeleteKey = regionNamePrefix + MetricsRegionServerSource.DELETE_KEY + suffix; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServer.java index 4c28763..e7282e9 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServer.java @@ -79,10 +79,22 @@ public class MetricsRegionServer { } public void updateDelete(long t) { + serverSource.updateDelete(t); + } + + public void updateDeleteBatch(long t) { if (t > 1000) { serverSource.incrSlowDelete(); } - serverSource.updateDelete(t); + serverSource.updateDeleteBatch(t); + } + + public void updateCheckAndDelete(long t) { + serverSource.updateCheckAndDelete(t); + } + + public void updateCheckAndPut(long t) { + serverSource.updateCheckAndPut(t); } public void updateGet(long t) { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java index 8bd1b5d..c17d141 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java @@ -1037,7 +1037,7 @@ public class RSRpcServices implements HBaseRPCErrorHandler, regionServer.metricsRegionServer.updatePut(after - before); } if (batchContainsDelete) { - regionServer.metricsRegionServer.updateDelete(after - before); + regionServer.metricsRegionServer.updateDeleteBatch(after - before); } } } @@ -1112,7 +1112,7 @@ public class RSRpcServices implements HBaseRPCErrorHandler, regionServer.metricsRegionServer.updatePut(after - before); } if (batchContainsDelete) { - regionServer.metricsRegionServer.updateDelete(after - before); + regionServer.metricsRegionServer.updateDeleteBatch(after - before); } } } @@ -2722,6 +2722,8 @@ public class RSRpcServices implements HBaseRPCErrorHandler, OperationQuota quota = null; RpcCallContext context = RpcServer.getCurrentCall(); ActivePolicyEnforcement spaceQuotaEnforcement = null; + MutationType type = null; + long before = EnvironmentEdgeManager.currentTime(); // Clear scanner so we are not holding on to reference across call. if (controller != null) { controller.setCellScanner(null); @@ -2739,7 +2741,7 @@ public class RSRpcServices implements HBaseRPCErrorHandler, long nonceGroup = request.hasNonceGroup() ? request.getNonceGroup() : HConstants.NO_NONCE; Result r = null; Boolean processed = null; - MutationType type = mutation.getMutateType(); + type = mutation.getMutateType(); quota = getRpcQuotaManager().checkQuota(region, OperationQuota.OperationType.MUTATE); spaceQuotaEnforcement = getSpaceQuotaManager().getActiveEnforcements(); @@ -2836,6 +2838,29 @@ public class RSRpcServices implements HBaseRPCErrorHandler, if (quota != null) { quota.close(); } + // Update metrics + if (regionServer.metricsRegionServer != null) { + long after = EnvironmentEdgeManager.currentTime(); + switch (type) { + case DELETE: + if (request.hasCondition()) { + regionServer.metricsRegionServer.updateCheckAndDelete(after - before); + } else { + regionServer.metricsRegionServer.updateDelete(after - before); + } + break; + case PUT: + if (request.hasCondition()) { + regionServer.metricsRegionServer.updateCheckAndPut(after - before); + } else { + regionServer.metricsRegionServer.updatePut(after - before); + } + break; + default: + break; + + } + } } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionServer.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionServer.java index 3a9d2ee..4966ff8 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionServer.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionServer.java @@ -111,8 +111,8 @@ public class TestMetricsRegionServer { rsm.updateAppend(1002); } for (int i=0; i < 13; i ++) { - rsm.updateDelete(13); - rsm.updateDelete(1003); + rsm.updateDeleteBatch(13); + rsm.updateDeleteBatch(1003); } for (int i=0; i < 14; i ++) { rsm.updateGet(14); @@ -127,11 +127,20 @@ public class TestMetricsRegionServer { rsm.updatePut(1006); } + for (int i=0; i < 17; i ++) { + rsm.updateDelete(17); + rsm.updateCheckAndDelete(17); + rsm.updateCheckAndPut(17); + } + HELPER.assertCounter("appendNumOps", 24, serverSource); - HELPER.assertCounter("deleteNumOps", 26, serverSource); + HELPER.assertCounter("deleteBatchNumOps", 26, serverSource); HELPER.assertCounter("getNumOps", 28, serverSource); HELPER.assertCounter("incrementNumOps", 30, serverSource); - HELPER.assertCounter("mutateNumOps", 32, serverSource); + HELPER.assertCounter("putNumOps", 32, serverSource); + HELPER.assertCounter("deleteNumOps", 17, serverSource); + HELPER.assertCounter("checkAndDeleteNumOps", 17, serverSource); + HELPER.assertCounter("checkAndPutNumOps", 17, serverSource); HELPER.assertCounter("slowAppendCount", 12, serverSource); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java index 128b010..4be76e6 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java @@ -260,7 +260,7 @@ public class TestRegionServerMetrics { assertCounter("writeRequestCount", writeRequests + 30); assertRegionMetrics("getCount", 10); - assertRegionMetrics("mutateCount", 31); + assertRegionMetrics("putCount", 31); doNGets(10, true); // true = batch diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRemoveRegionMetrics.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRemoveRegionMetrics.java index 85699ef..b007161 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRemoveRegionMetrics.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRemoveRegionMetrics.java @@ -114,7 +114,7 @@ public class TestRemoveRegionMetrics { "_region_" + regionInfo.getEncodedName()+ "_metric"; - metricsHelper.assertCounter(prefix + "_mutateCount", 1, currentAgg); + metricsHelper.assertCounter(prefix + "_putCount", 1, currentAgg); try { @@ -129,7 +129,7 @@ public class TestRemoveRegionMetrics { .getMetrics() .getSource() .getAggregateSource(); - metricsHelper.assertCounter(prefix + "_mutateCount", 0, destAgg); + metricsHelper.assertCounter(prefix + "_putCount", 0, destAgg); } } -- 2.1.2