commit 0d17c5abeb3d8b30ed849b040e872a5b4cee5707 Author: nspiegelberg Date: 38 minutes ago [jira] [HBASE-5072] Support Max Value for Per-Store Metrics Summary: We were bit in our multi-tenant cluster because one of our Stores encountered a bug and grew its StoreFile count. We didn't notice this because the StoreFile count currently reported by the RegionServer is an average of all Stores in the region. For the per-Store metrics, we should also record the max so we can notice outliers. Test Plan: - mvn test -Dtest=TestRegionServerMetrics Reviewers: JIRA, mbautin, Kannan Reviewed By: Kannan CC: stack, nspiegelberg, mbautin, Kannan Differential Revision: 945 diff --git src/main/java/org/apache/hadoop/hbase/regionserver/metrics/SchemaMetrics.java src/main/java/org/apache/hadoop/hbase/regionserver/metrics/SchemaMetrics.java index 3e587c3..08c5cfb 100644 --- src/main/java/org/apache/hadoop/hbase/regionserver/metrics/SchemaMetrics.java +++ src/main/java/org/apache/hadoop/hbase/regionserver/metrics/SchemaMetrics.java @@ -245,6 +245,7 @@ public class SchemaMetrics { private final String[] bloomMetricNames = new String[2]; private final String[] storeMetricNames = new String[NUM_STORE_METRIC_TYPES]; + private final String[] storeMetricNamesMax = new String[NUM_STORE_METRIC_TYPES]; private SchemaMetrics(final String tableName, final String cfName) { String metricPrefix = SchemaMetrics.generateSchemaMetricsPrefix( @@ -290,8 +291,9 @@ public class SchemaMetrics { } for (StoreMetricType storeMetric : StoreMetricType.values()) { - storeMetricNames[storeMetric.ordinal()] = metricPrefix + - storeMetric.toString(); + String coreName = metricPrefix + storeMetric.toString(); + storeMetricNames[storeMetric.ordinal()] = coreName; + storeMetricNamesMax[storeMetric.ordinal()] = coreName + ".max"; } } @@ -390,13 +392,22 @@ public class SchemaMetrics { public void accumulateStoreMetric(final Map tmpMap, StoreMetricType storeMetricType, double val) { final String key = getStoreMetricName(storeMetricType); - if (tmpMap.get(key) != null) { - tmpMap.get(key).add(val); - } else { + if (tmpMap.get(key) == null) { tmpMap.put(key, new MutableDouble(val)); + } else { + tmpMap.get(key).add(val); } - if (this != ALL_SCHEMA_METRICS) { + if (this == ALL_SCHEMA_METRICS) { + // also compute the max value across all Stores on this server + final String maxKey = getStoreMetricNameMax(storeMetricType); + MutableDouble cur = tmpMap.get(maxKey); + if (cur == null) { + tmpMap.put(maxKey, new MutableDouble(val)); + } else if (cur.doubleValue() < val) { + cur.setValue(val); + } + } else { ALL_SCHEMA_METRICS.accumulateStoreMetric(tmpMap, storeMetricType, val); } } @@ -405,6 +416,10 @@ public class SchemaMetrics { return storeMetricNames[storeMetricType.ordinal()]; } + public String getStoreMetricNameMax(StoreMetricType storeMetricType) { + return storeMetricNamesMax[storeMetricType.ordinal()]; + } + /** * Update a metric that does not get reset on every poll. * @param storeMetricType the store metric to update diff --git src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java index 3447aeb..6560672 100644 --- src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java +++ src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java @@ -116,9 +116,15 @@ public class TestRegionServerMetrics { for (String cf : FAMILIES) { SchemaMetrics schemaMetrics = SchemaMetrics.getInstance(TABLE_NAME, cf); - assertStoreMetricEquals(NUM_FLUSHES * NUM_REGIONS, - schemaMetrics, StoreMetricType.STORE_FILE_COUNT); + assertStoreMetricEquals(NUM_FLUSHES * NUM_REGIONS, schemaMetrics, + StoreMetricType.STORE_FILE_COUNT); } + + // ensure that the max value is also maintained + final String storeMetricName = ALL_METRICS + .getStoreMetricNameMax(StoreMetricType.STORE_FILE_COUNT); + assertEquals("Invalid value for store metric " + storeMetricName, + NUM_FLUSHES, HRegion.getNumericMetric(storeMetricName)); }