diff --git src/main/java/org/apache/hadoop/hbase/ipc/HBaseRpcMetrics.java src/main/java/org/apache/hadoop/hbase/ipc/HBaseRpcMetrics.java index bc897d9..bddcc14 100644 --- src/main/java/org/apache/hadoop/hbase/ipc/HBaseRpcMetrics.java +++ src/main/java/org/apache/hadoop/hbase/ipc/HBaseRpcMetrics.java @@ -20,15 +20,19 @@ package org.apache.hadoop.hbase.ipc; +import java.util.HashMap; +import java.util.Map; +import java.util.Map.Entry; + import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hbase.ipc.VersionedProtocol; +import org.apache.hadoop.hbase.metrics.histogram.MetricsTimeVaryingHistogram; import org.apache.hadoop.metrics.MetricsContext; import org.apache.hadoop.metrics.MetricsRecord; import org.apache.hadoop.metrics.MetricsUtil; import org.apache.hadoop.metrics.Updater; import org.apache.hadoop.metrics.util.*; - import java.lang.reflect.Method; /** @@ -46,6 +50,7 @@ import java.lang.reflect.Method; public class HBaseRpcMetrics implements Updater { public static final String NAME_DELIM = "$"; private final MetricsRegistry registry = new MetricsRegistry(); + private final Map methodHistograms = new HashMap(); private final MetricsRecord metricsRecord; private static Log LOG = LogFactory.getLog(HBaseRpcMetrics.class); private final HBaseRPCStatistics rpcStatistics; @@ -103,8 +108,14 @@ public class HBaseRpcMetrics implements Updater { private void initMethods(Class protocol) { for (Method m : protocol.getDeclaredMethods()) { - if (get(m.getName()) == null) + if (get(m.getName()) == null) { create(m.getName()); + // we report percentile only for methods defined in HReginInterface + if (protocol == HRegionInterface.class) { + methodHistograms.put(m.getName(), new MetricsTimeVaryingHistogram(m.getName(), null)); + LOG.debug("create MetricTimeVaryingHistogram for methodName=" + m.getName()); + } + } } } @@ -114,7 +125,7 @@ public class HBaseRpcMetrics implements Updater { private MetricsTimeVaryingRate create(String key) { return new MetricsTimeVaryingRate(key, this.registry); } - + public void inc(String name, int amt) { MetricsTimeVaryingRate m = get(name); if (m == null) { @@ -123,6 +134,12 @@ public class HBaseRpcMetrics implements Updater { return; // ignore methods that dont exist. } m.inc(amt); + + // update histogram + MetricsTimeVaryingHistogram histogram = methodHistograms.get(name); + if (histogram != null) { + histogram.update(amt); + } } /** @@ -204,6 +221,12 @@ public class HBaseRpcMetrics implements Updater { * Push the metrics to the monitoring subsystem on doUpdate() call. */ public void doUpdates(final MetricsContext context) { + for (Entry entry : methodHistograms.entrySet()) { + MetricsTimeVaryingHistogram.registerTimeVaryingHistogramMetric(entry.getKey(), + entry.getValue(), this.registry); + entry.getValue().pushMetric(metricsRecord); + } + // Both getMetricsList() and pushMetric() are thread-safe for (MetricsBase m : registry.getMetricsList()) { m.pushMetric(metricsRecord); diff --git src/main/java/org/apache/hadoop/hbase/metrics/histogram/MetricsHistogram.java src/main/java/org/apache/hadoop/hbase/metrics/histogram/MetricsHistogram.java index 392cbf9..4fc8a41 100644 --- src/main/java/org/apache/hadoop/hbase/metrics/histogram/MetricsHistogram.java +++ src/main/java/org/apache/hadoop/hbase/metrics/histogram/MetricsHistogram.java @@ -48,7 +48,8 @@ public class MetricsHistogram extends MetricsBase { public static final String SEVENTY_FIFTH_PERCENTILE_METRIC_NAME = "_75th_percentile"; public static final String NINETY_FIFTH_PERCENTILE_METRIC_NAME = "_95th_percentile"; public static final String NINETY_NINETH_PERCENTILE_METRIC_NAME = "_99th_percentile"; - + public static final String NINETY_NINE_POINT_NINETH_PERCENTILE_METRIC_NAME = "_999th_percentile"; + /** * Constructor to create a new histogram metric * @param nam the name to publish the metric under @@ -235,5 +236,7 @@ public class MetricsHistogram extends MetricsBase { (float) s.get95thPercentile()); mr.setMetric(getName() + NINETY_NINETH_PERCENTILE_METRIC_NAME, (float) s.get99thPercentile()); + mr.setMetric(getName() + NINETY_NINE_POINT_NINETH_PERCENTILE_METRIC_NAME, + (float) s.get999thPercentile()); } } diff --git src/main/java/org/apache/hadoop/hbase/metrics/histogram/MetricsTimeVaryingHistogram.java src/main/java/org/apache/hadoop/hbase/metrics/histogram/MetricsTimeVaryingHistogram.java new file mode 100644 index 0000000..f5da049 --- /dev/null +++ src/main/java/org/apache/hadoop/hbase/metrics/histogram/MetricsTimeVaryingHistogram.java @@ -0,0 +1,126 @@ +package org.apache.hadoop.hbase.metrics.histogram; + +import org.apache.hadoop.metrics.MetricsRecord; +import org.apache.hadoop.metrics.util.MetricsLongValue; +import org.apache.hadoop.metrics.util.MetricsRegistry; + +import com.yammer.metrics.stats.Snapshot; + +/** + * The MetricsTimeVaryingHistogram class is for a MetricsHistogram that naturally + * varies over time (e.g. latency of operation). The MetricsHistogram is published + * at the end of each interval and then clear. Hence the counter has the histogram + * in the current interval. + */ +public class MetricsTimeVaryingHistogram extends MetricsHistogram { + public static final String HISTOGRAM = "histogram"; + + public MetricsTimeVaryingHistogram(final String name, final MetricsRegistry registry, + final String description, boolean forwardBiased) { + super(name, registry, description, forwardBiased); + } + + public MetricsTimeVaryingHistogram(final String name, MetricsRegistry registry, + final String description) { + super(name, registry, description); + } + + public MetricsTimeVaryingHistogram(final String name, MetricsRegistry registry) { + super(name, registry); + } + + @Override + public void pushMetric(MetricsRecord mr) { + super.pushMetric(mr); + clear(); + } + + // Some method names will end with '_', such as methods in OperationMetrics; + // while others may not, such as methods in HBaseRpcMetrics. This method + // will adapt the two cases. + public static String getHistogramMetricName(String methodName, String suffix) { + if (methodName.endsWith("_")) { + return methodName + HISTOGRAM + suffix; + } + return methodName + "_" + HISTOGRAM + suffix; + } + + public static void registerTimeVaryingHistogramMetric(String methodName, + MetricsTimeVaryingHistogram histogram, MetricsRegistry registry) { + String countName = getHistogramMetricName(methodName, MetricsHistogram.NUM_OPS_METRIC_NAME); + MetricsLongValue metricCount = (MetricsLongValue)registry.get(countName); + if (metricCount == null) { + metricCount = new MetricsLongValue(countName, registry); + } + metricCount.set(histogram.getCount()); + + String minName = getHistogramMetricName(methodName, MetricsHistogram.MIN_METRIC_NAME); + MetricsLongValue metricMin = (MetricsLongValue)registry.get(minName); + if (metricMin == null) { + metricMin = new MetricsLongValue(minName, registry); + } + metricMin.set(histogram.getMin()); + + String maxName = getHistogramMetricName(methodName, MetricsHistogram.MAX_METRIC_NAME); + MetricsLongValue metricMax = (MetricsLongValue)registry.get(maxName); + if (metricMax == null) { + metricMax = new MetricsLongValue(maxName, registry); + } + metricMax.set(histogram.getMax()); + + String meanName = getHistogramMetricName(methodName, MetricsHistogram.MEAN_METRIC_NAME); + MetricsLongValue metricMean = (MetricsLongValue)registry.get(meanName); + if (metricMean == null) { + metricMean = new MetricsLongValue(meanName, registry); + } + metricMean.set((long)histogram.getMean()); + + String stdDevName = getHistogramMetricName(methodName, MetricsHistogram.STD_DEV_METRIC_NAME); + MetricsLongValue metricStdDev = (MetricsLongValue)registry.get(stdDevName); + if (metricStdDev == null) { + metricStdDev = new MetricsLongValue(stdDevName, registry); + } + metricStdDev.set((long)histogram.getStdDev()); + + Snapshot snapshot = histogram.getSnapshot(); + + String medianName = getHistogramMetricName(methodName, MetricsHistogram.MEDIAN_METRIC_NAME); + MetricsLongValue metricMedian = (MetricsLongValue)registry.get(medianName); + if (metricMedian == null) { + metricMedian = new MetricsLongValue(medianName, registry); + } + metricMedian.set((long)snapshot.getMedian()); + + String _75thPercentileName = getHistogramMetricName(methodName, + MetricsHistogram.SEVENTY_FIFTH_PERCENTILE_METRIC_NAME); + MetricsLongValue metric75thPercentile = (MetricsLongValue)registry.get(_75thPercentileName); + if (metric75thPercentile == null) { + metric75thPercentile = new MetricsLongValue(_75thPercentileName, registry); + } + metric75thPercentile.set((long)snapshot.get75thPercentile()); + + String _95thPercentileName = getHistogramMetricName(methodName, + MetricsHistogram.NINETY_FIFTH_PERCENTILE_METRIC_NAME); + MetricsLongValue metric95thPercentile = (MetricsLongValue)registry.get(_95thPercentileName); + if (metric95thPercentile == null) { + metric95thPercentile = new MetricsLongValue(_95thPercentileName, registry); + } + metric95thPercentile.set((long)snapshot.get95thPercentile()); + + String _99thPercentileName = getHistogramMetricName(methodName, + MetricsHistogram.NINETY_NINETH_PERCENTILE_METRIC_NAME); + MetricsLongValue metric99thPercentile = (MetricsLongValue)registry.get(_99thPercentileName); + if (metric99thPercentile == null) { + metric99thPercentile = new MetricsLongValue(_99thPercentileName, registry); + } + metric99thPercentile.set((long)snapshot.get99thPercentile()); + + String _999thPercentileName = getHistogramMetricName(methodName, + MetricsHistogram.NINETY_NINE_POINT_NINETH_PERCENTILE_METRIC_NAME); + MetricsLongValue metric999thPercentile = (MetricsLongValue)registry.get(_999thPercentileName); + if (metric999thPercentile == null) { + metric999thPercentile = new MetricsLongValue(_999thPercentileName, registry); + } + metric999thPercentile.set((long)snapshot.get999thPercentile()); + } +} diff --git src/test/java/org/apache/hadoop/hbase/metrics/TestMetricsTimeVaryingHistogram.java src/test/java/org/apache/hadoop/hbase/metrics/TestMetricsTimeVaryingHistogram.java new file mode 100644 index 0000000..54327e9 --- /dev/null +++ src/test/java/org/apache/hadoop/hbase/metrics/TestMetricsTimeVaryingHistogram.java @@ -0,0 +1,91 @@ +package org.apache.hadoop.hbase.metrics; + +import org.apache.hadoop.hbase.metrics.histogram.MetricsTimeVaryingHistogram; +import org.apache.hadoop.metrics.MetricsContext; +import org.apache.hadoop.metrics.MetricsRecord; +import org.apache.hadoop.metrics.MetricsUtil; +import org.apache.hadoop.metrics.util.MetricsLongValue; +import org.apache.hadoop.metrics.util.MetricsRegistry; +import org.junit.Assert; +import org.junit.Test; + +public class TestMetricsTimeVaryingHistogram { + @Test + public void testTimeVaring() { + MetricsContext context = MetricsUtil.getContext("test"); + MetricsRecord metricsRecord = MetricsUtil.createRecord(context, "test"); + + MetricsTimeVaryingHistogram h = new MetricsTimeVaryingHistogram("testHistogram", null); + + for (int i = 0; i < 2; ++i) { + for (int j = 1; j <= 100; j++) { + h.update(j); + } + Assert.assertEquals(100, h.getCount()); + Assert.assertEquals(1, h.getMin()); + Assert.assertEquals(100, h.getMax()); + // push metric and clear counters for last interval + h.pushMetric(metricsRecord); + } + } + + @Test + public void testGetHistogramMetricName() { + String method = "methodA"; + String suffix = "_suffix"; + Assert.assertEquals("methodA_histogram_suffix", + MetricsTimeVaryingHistogram.getHistogramMetricName(method, suffix)); + + method = "methodA_"; + Assert.assertEquals("methodA_histogram_suffix", + MetricsTimeVaryingHistogram.getHistogramMetricName(method, suffix)); + } + + @Test + public void testRegisterTimeVaryingHistogramMetric() { + MetricsTimeVaryingHistogram histogram = new MetricsTimeVaryingHistogram("TestHistogram", null); + for (int i = 1; i <= 100; ++i) { + histogram.update(i); + } + String methodName = "testMethod"; + MetricsRegistry registry = new MetricsRegistry(); + MetricsTimeVaryingHistogram.registerTimeVaryingHistogramMetric(methodName, histogram, registry); + Assert.assertNotNull(registry.get("testMethod_histogram_num_ops")); + Assert.assertEquals(MetricsLongValue.class, registry.get("testMethod_histogram_num_ops").getClass()); + Assert.assertEquals(100, ((MetricsLongValue)registry.get("testMethod_histogram_num_ops")).get()); + + Assert.assertNotNull(registry.get("testMethod_histogram_min")); + Assert.assertEquals(MetricsLongValue.class, registry.get("testMethod_histogram_min").getClass()); + Assert.assertEquals(1, ((MetricsLongValue)registry.get("testMethod_histogram_min")).get()); + + Assert.assertNotNull(registry.get("testMethod_histogram_max")); + Assert.assertEquals(MetricsLongValue.class, registry.get("testMethod_histogram_max").getClass()); + Assert.assertEquals(100, ((MetricsLongValue)registry.get("testMethod_histogram_max")).get()); + + Assert.assertNotNull(registry.get("testMethod_histogram_mean")); + Assert.assertEquals(MetricsLongValue.class, registry.get("testMethod_histogram_mean").getClass()); + Assert.assertEquals(50, ((MetricsLongValue)registry.get("testMethod_histogram_mean")).get(), 0.0001f); + + Assert.assertNotNull(registry.get("testMethod_histogram_std_dev")); + Assert.assertEquals(MetricsLongValue.class, registry.get("testMethod_histogram_std_dev").getClass()); + Assert.assertNotNull(registry.get("testMethod_histogram_median")); + Assert.assertEquals(MetricsLongValue.class, registry.get("testMethod_histogram_median").getClass()); + Assert.assertEquals(50, ((MetricsLongValue)registry.get("testMethod_histogram_median")).get(), 0.0001f); + + Assert.assertNotNull(registry.get("testMethod_histogram_75th_percentile")); + Assert.assertEquals(MetricsLongValue.class, registry.get("testMethod_histogram_75th_percentile").getClass()); + Assert.assertEquals(75, ((MetricsLongValue)registry.get("testMethod_histogram_75th_percentile")).get(), 0.0001f); + + Assert.assertNotNull(registry.get("testMethod_histogram_95th_percentile")); + Assert.assertEquals(MetricsLongValue.class, registry.get("testMethod_histogram_95th_percentile").getClass()); + Assert.assertEquals(95, ((MetricsLongValue)registry.get("testMethod_histogram_95th_percentile")).get(), 0.0001f); + + Assert.assertNotNull(registry.get("testMethod_histogram_99th_percentile")); + Assert.assertEquals(MetricsLongValue.class, registry.get("testMethod_histogram_99th_percentile").getClass()); + Assert.assertEquals(99, ((MetricsLongValue)registry.get("testMethod_histogram_99th_percentile")).get(), 0.0001f); + + Assert.assertNotNull(registry.get("testMethod_histogram_999th_percentile")); + Assert.assertEquals(MetricsLongValue.class, registry.get("testMethod_histogram_999th_percentile").getClass()); + Assert.assertEquals(100, ((MetricsLongValue)registry.get("testMethod_histogram_999th_percentile")).get(), 0.0001f); + } +}