diff --git src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java index 7858846..7ad6e49 100644 --- src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java +++ src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java @@ -966,6 +966,7 @@ public class HRegion implements HeapSize { // , Writable{ status.setStatus("Running coprocessor post-close hooks"); this.coprocessorHost.postClose(abort); } + this.opMetrics.closeMetrics(); status.markComplete("Closed"); LOG.info("Closed " + this); return result; diff --git src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java index 61a5988..c0b1dbb 100644 --- src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java +++ src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java @@ -2782,6 +2782,8 @@ public class HRegionServer extends RegionServer public boolean removeFromOnlineRegions(final String encodedName) { HRegion toReturn = null; toReturn = this.onlineRegions.remove(encodedName); + //Clear all of the dynamic metrics as they are now probably useless + this.dynamicMetrics.clear(); return toReturn != null; } diff --git src/main/java/org/apache/hadoop/hbase/regionserver/metrics/OperationMetrics.java src/main/java/org/apache/hadoop/hbase/regionserver/metrics/OperationMetrics.java index ac4e57f..305dfab 100644 --- src/main/java/org/apache/hadoop/hbase/regionserver/metrics/OperationMetrics.java +++ src/main/java/org/apache/hadoop/hbase/regionserver/metrics/OperationMetrics.java @@ -29,7 +29,6 @@ import org.apache.hadoop.hbase.client.Get; import org.apache.hadoop.hbase.client.HTable; import org.apache.hadoop.hbase.client.Increment; import org.apache.hadoop.hbase.client.Put; -import org.apache.hadoop.hbase.regionserver.HRegion; import org.apache.hadoop.hbase.util.Bytes; /** @@ -64,12 +63,12 @@ public class OperationMetrics { * @param conf The Configuration of the HRegion reporting operations coming in. * @param regionInfo The region info */ - public OperationMetrics(Configuration conf, HRegionInfo regionInfo) { + public OperationMetrics(Configuration conf, HRegionInfo regionInfo) { // Configure SchemaMetrics before trying to create a RegionOperationMetrics instance as // RegionOperationMetrics relies on SchemaMetrics to do naming. if (conf != null) { SchemaMetrics.configureGlobally(conf); - + this.conf = conf; if (regionInfo != null) { this.tableName = regionInfo.getTableNameAsString(); @@ -172,6 +171,13 @@ public class OperationMetrics { public void updateDeleteMetrics(Set columnFamilies, long value) { doUpdateTimeVarying(columnFamilies, DELETE_KEY, value); } + + /** + * This deletes all old metrics this instance has ever created or updated. + */ + public void closeMetrics() { + RegionMetricsStorage.clear(); + } /** * Method to send updates for cf and region metrics. This is the normal method @@ -199,7 +205,8 @@ public class OperationMetrics { private void doSafeIncTimeVarying(String prefix, String key, long value) { if (conf.getBoolean(CONF_KEY, true)) { if (prefix != null && !prefix.isEmpty() && key != null && !key.isEmpty()) { - RegionMetricsStorage.incrTimeVaryingMetric(prefix + key, value); + String m = prefix + key; + RegionMetricsStorage.incrTimeVaryingMetric(m, value); } } } diff --git src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionMetricsStorage.java src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionMetricsStorage.java index 416e495..5d6b252 100644 --- src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionMetricsStorage.java +++ src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionMetricsStorage.java @@ -127,4 +127,9 @@ public class RegionMetricsStorage { return m.get(); } + public static void clear() { + timeVaryingMetrics.clear(); + numericMetrics.clear(); + numericPersistentMetrics.clear(); + } } diff --git src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerDynamicMetrics.java src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerDynamicMetrics.java index eb733a0..3dfa91d 100644 --- src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerDynamicMetrics.java +++ src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerDynamicMetrics.java @@ -20,7 +20,9 @@ package org.apache.hadoop.hbase.regionserver.metrics; +import java.lang.reflect.Field; import java.lang.reflect.Method; +import java.util.Map; import java.util.Map.Entry; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicInteger; @@ -52,12 +54,18 @@ import org.apache.hadoop.metrics.util.MetricsTimeVaryingRate; */ @InterfaceAudience.Private public class RegionServerDynamicMetrics implements Updater { + private static final String UNABLE_TOCLEAR = "Unable to clear RegionServerDynamicMetrics"; + private MetricsRecord metricsRecord; private MetricsContext context; private final RegionServerDynamicStatistics rsDynamicStatistics; private Method updateMbeanInfoIfMetricsListChanged = null; private static final Log LOG = LogFactory.getLog(RegionServerDynamicStatistics.class); + + private boolean reflectionInitialized = false; + private Field recordMetricMapField; + private Field registryMetricMapField; /** * The metrics variables are public: @@ -126,6 +134,51 @@ public class RegionServerDynamicMetrics implements Updater { m.inc(numOps, amt); } } + + @SuppressWarnings("rawtypes") + public void clear() { + + //Try and get the private fields from the Hadoop Metrics that keep registered metrics around + if (!this.reflectionInitialized) { + + try { + this.recordMetricMapField = this.metricsRecord.getClass().getDeclaredField("metricTable"); + this.recordMetricMapField.setAccessible(true); + } catch (SecurityException e) { + LOG.debug(UNABLE_TOCLEAR); + } catch (NoSuchFieldException e) { + LOG.debug(UNABLE_TOCLEAR); + } + + try { + this.registryMetricMapField = this.registry.getClass().getDeclaredField("metricsList"); + this.registryMetricMapField.setAccessible(true); + } catch (SecurityException e) { + LOG.debug(UNABLE_TOCLEAR); + } catch (NoSuchFieldException e) { + LOG.debug(UNABLE_TOCLEAR); + } + + this.reflectionInitialized = true; + } + + + //If we found both fields then try and clear the maps. + if (this.recordMetricMapField != null && this.registryMetricMapField != null) { + try { + Map recordMap = (Map) this.recordMetricMapField.get(this.metricsRecord); + recordMap.clear(); + Map registryMap = (Map) this.registryMetricMapField.get(this.registry); + registryMap.clear(); + } catch (IllegalArgumentException e) { + LOG.debug(UNABLE_TOCLEAR); + } catch (IllegalAccessException e) { + LOG.debug(UNABLE_TOCLEAR); + } + } else { + LOG.debug(UNABLE_TOCLEAR); + } + } /** * Push the metrics to the monitoring subsystem on doUpdate() call. diff --git src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java index 6defa73..aa5ca37 100644 --- src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java +++ src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java @@ -49,6 +49,7 @@ import org.junit.Before; import org.junit.Test; import org.junit.experimental.categories.Category; + /** * Test metrics incremented on region server operations. */ @@ -197,6 +198,32 @@ public class TestRegionServerMetrics { } @Test + public void testRemoveRegionMetrics() throws IOException, InterruptedException { + String cf = "REMOVECF"; + HTable hTable = TEST_UTIL.createTable(TABLE_NAME.getBytes(), cf.getBytes()); + HRegionInfo[] regionInfos = + hTable.getRegionLocations().keySet() + .toArray(new HRegionInfo[hTable.getRegionLocations().keySet().size()]); + + String regionName = regionInfos[0].getEncodedName(); + + // Do some operations so there are metrics. + Put pOne = new Put("TEST".getBytes()); + pOne.add(cf.getBytes(), "test".getBytes(), "test".getBytes()); + hTable.put(pOne); + + Get g = new Get("TEST".getBytes()); + g.addFamily(cf.getBytes()); + hTable.get(g); + assertTimeVaryingMetricCount(1, TABLE_NAME, cf, regionName, "get_"); + HBaseAdmin admin = TEST_UTIL.getHBaseAdmin(); + admin.disableTable(TABLE_NAME.getBytes()); + admin.deleteTable(TABLE_NAME.getBytes()); + + assertTimeVaryingMetricCount(0, TABLE_NAME, cf, regionName, "get_"); + } + + @Test public void testMultipleRegions() throws IOException, InterruptedException { TEST_UTIL.createRandomTable(