Index: src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java =================================================================== --- src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java (revision 1330481) +++ src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java (working copy) @@ -33,6 +33,7 @@ import org.apache.hadoop.hbase.client.Append; import org.apache.hadoop.hbase.client.Delete; import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.HBaseAdmin; import org.apache.hadoop.hbase.client.HTable; import org.apache.hadoop.hbase.client.Increment; import org.apache.hadoop.hbase.client.Put; @@ -47,6 +48,7 @@ import org.junit.Test; import org.junit.experimental.categories.Category; + /** * Test metrics incremented on region server operations. */ @@ -200,6 +202,32 @@ } @Test + public void testRemoveRegionMetrics() throws IOException, InterruptedException { + String cf = "REMOVECF"; + HTable hTable = TEST_UTIL.createTable(TABLE_NAME.getBytes(), cf.getBytes()); + HRegionInfo[] regionInfos = + hTable.getRegionLocations().keySet() + .toArray(new HRegionInfo[hTable.getRegionLocations().keySet().size()]); + + String regionName = regionInfos[0].getEncodedName(); + + // Do some operations so there are metrics. + Put pOne = new Put("TEST".getBytes()); + pOne.add(cf.getBytes(), "test".getBytes(), "test".getBytes()); + hTable.put(pOne); + + Get g = new Get("TEST".getBytes()); + g.addFamily(cf.getBytes()); + hTable.get(g); + assertTimeVaryingMetricCount(1, TABLE_NAME, cf, regionName, "get_"); + HBaseAdmin admin = TEST_UTIL.getHBaseAdmin(); + admin.disableTable(TABLE_NAME.getBytes()); + admin.deleteTable(TABLE_NAME.getBytes()); + + assertTimeVaryingMetricCount(0, TABLE_NAME, cf, regionName, "get_"); + } + + @Test public void testMultipleRegions() throws IOException, InterruptedException { TEST_UTIL.createRandomTable( Index: src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionMetricsStorage.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionMetricsStorage.java (revision 1330481) +++ src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionMetricsStorage.java (working copy) @@ -127,4 +127,9 @@ return m.get(); } + public static void clear() { + timeVaryingMetrics.clear(); + numericMetrics.clear(); + numericPersistentMetrics.clear(); + } } Index: src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerDynamicMetrics.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerDynamicMetrics.java (revision 1330481) +++ src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerDynamicMetrics.java (working copy) @@ -20,7 +20,9 @@ package org.apache.hadoop.hbase.regionserver.metrics; +import java.lang.reflect.Field; import java.lang.reflect.Method; +import java.util.Map; import java.util.Map.Entry; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicInteger; @@ -50,12 +52,18 @@ * */ public class RegionServerDynamicMetrics implements Updater { + private static final String UNABLE_TOCLEAR = "Unable to clear RegionServerDynamicMetrics"; + private MetricsRecord metricsRecord; private MetricsContext context; private final RegionServerDynamicStatistics rsDynamicStatistics; private Method updateMbeanInfoIfMetricsListChanged = null; private static final Log LOG = LogFactory.getLog(RegionServerDynamicStatistics.class); + + private boolean reflectionInitialized = false; + private Field recordMetricMapField; + private Field registryMetricMapField; /** * The metrics variables are public: @@ -124,7 +132,52 @@ m.inc(numOps, amt); } } + + @SuppressWarnings("rawtypes") + public void clear() { + + //Try and get the private fields from the Hadoop Metrics that keep registered metrics around + if (!this.reflectionInitialized) { + try { + this.recordMetricMapField = this.metricsRecord.getClass().getDeclaredField("metricTable"); + this.recordMetricMapField.setAccessible(true); + } catch (SecurityException e) { + LOG.debug(UNABLE_TOCLEAR); + } catch (NoSuchFieldException e) { + LOG.debug(UNABLE_TOCLEAR); + } + + try { + this.registryMetricMapField = this.registry.getClass().getDeclaredField("metricsList"); + this.registryMetricMapField.setAccessible(true); + } catch (SecurityException e) { + LOG.debug(UNABLE_TOCLEAR); + } catch (NoSuchFieldException e) { + LOG.debug(UNABLE_TOCLEAR); + } + + this.reflectionInitialized = true; + } + + + //If we found both fields then try and clear the maps. + if (this.recordMetricMapField != null && this.registryMetricMapField != null) { + try { + Map recordMap = (Map) this.recordMetricMapField.get(this.metricsRecord); + recordMap.clear(); + Map registryMap = (Map) this.registryMetricMapField.get(this.registry); + registryMap.clear(); + } catch (IllegalArgumentException e) { + LOG.debug(UNABLE_TOCLEAR); + } catch (IllegalAccessException e) { + LOG.debug(UNABLE_TOCLEAR); + } + } else { + LOG.debug(UNABLE_TOCLEAR); + } + } + /** * Push the metrics to the monitoring subsystem on doUpdate() call. * @param context ctx Index: src/main/java/org/apache/hadoop/hbase/regionserver/metrics/OperationMetrics.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/regionserver/metrics/OperationMetrics.java (revision 1330481) +++ src/main/java/org/apache/hadoop/hbase/regionserver/metrics/OperationMetrics.java (working copy) @@ -29,7 +29,6 @@ import org.apache.hadoop.hbase.client.HTable; import org.apache.hadoop.hbase.client.Increment; import org.apache.hadoop.hbase.client.Put; -import org.apache.hadoop.hbase.regionserver.HRegion; import org.apache.hadoop.hbase.util.Bytes; /** @@ -64,12 +63,12 @@ * @param conf The Configuration of the HRegion reporting operations coming in. * @param regionInfo The region info */ - public OperationMetrics(Configuration conf, HRegionInfo regionInfo) { + public OperationMetrics(Configuration conf, HRegionInfo regionInfo) { // Configure SchemaMetrics before trying to create a RegionOperationMetrics instance as // RegionOperationMetrics relies on SchemaMetrics to do naming. if (conf != null) { SchemaMetrics.configureGlobally(conf); - + this.conf = conf; if (regionInfo != null) { this.tableName = regionInfo.getTableNameAsString(); @@ -172,6 +171,13 @@ public void updateDeleteMetrics(Set columnFamilies, long value) { doUpdateTimeVarying(columnFamilies, DELETE_KEY, value); } + + /** + * This deletes all old metrics this instance has ever created or updated. + */ + public void closeMetrics() { + RegionMetricsStorage.clear(); + } /** * Method to send updates for cf and region metrics. This is the normal method @@ -199,7 +205,8 @@ private void doSafeIncTimeVarying(String prefix, String key, long value) { if (conf.getBoolean(CONF_KEY, true)) { if (prefix != null && !prefix.isEmpty() && key != null && !key.isEmpty()) { - RegionMetricsStorage.incrTimeVaryingMetric(prefix + key, value); + String m = prefix + key; + RegionMetricsStorage.incrTimeVaryingMetric(m, value); } } } Index: src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java (revision 1330481) +++ src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java (working copy) @@ -926,6 +926,7 @@ status.setStatus("Running coprocessor post-close hooks"); this.coprocessorHost.postClose(abort); } + this.opMetrics.closeMetrics(); status.markComplete("Closed"); LOG.info("Closed " + this); return result; Index: src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java (revision 1330481) +++ src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java (working copy) @@ -3019,6 +3019,8 @@ public boolean removeFromOnlineRegions(final String encodedName) { HRegion toReturn = null; toReturn = this.onlineRegions.remove(encodedName); + //Clear all of the dynamic metrics as they are now probably useless + this.dynamicMetrics.clear(); return toReturn != null; }