From f9adb9bf472cf96b5ec0112525496a852733cfc2 Mon Sep 17 00:00:00 2001 From: Elliott Clark Date: Wed, 29 Jul 2015 15:04:46 -0700 Subject: [PATCH] HBASE-14166 Per-Region metrics can be stale --- .../regionserver/MetricsRegionAggregateSource.java | 3 + .../hbase/regionserver/MetricsRegionWrapper.java | 2 + .../MetricsRegionAggregateSourceImpl.java | 46 ++++++-- .../regionserver/MetricsRegionSourceImpl.java | 37 +++--- .../hadoop/metrics2/impl/JmxCacheBuster.java | 30 +++-- .../metrics2/lib/DefaultMetricsSystemHelper.java | 55 +++++++++ .../metrics2/lib/DynamicMetricsRegistry.java | 19 ++- .../hadoop/metrics2/lib/MetricsExecutorImpl.java | 8 +- .../regionserver/TestMetricsRegionSourceImpl.java | 5 + .../regionserver/MetricsRegionWrapperImpl.java | 5 + .../regionserver/MetricsRegionWrapperStub.java | 5 + .../regionserver/TestRemoveRegionMetrics.java | 131 +++++++++++++++++++++ 12 files changed, 305 insertions(+), 41 deletions(-) create mode 100644 hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/DefaultMetricsSystemHelper.java create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRemoveRegionMetrics.java diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionAggregateSource.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionAggregateSource.java index 0ef74a9..578ce49 100644 --- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionAggregateSource.java +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionAggregateSource.java @@ -46,6 +46,9 @@ public interface MetricsRegionAggregateSource extends BaseSource { */ String METRICS_JMX_CONTEXT = "RegionServer,sub=" + METRICS_NAME; + String NUM_REGIONS = "numRegions"; + String NUMBER_OF_REGIONS_DESC = "Number of regions in the metrics system"; + /** * Register a MetricsRegionSource as being open. * diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapper.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapper.java index 7d61f81..cfc0b66 100644 --- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapper.java +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapper.java @@ -84,6 +84,8 @@ public interface MetricsRegionWrapper { long getNumCompactionsCompleted(); + int getRegionHashCode(); + /** * Get the time spent by coprocessors in this region. */ diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionAggregateSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionAggregateSourceImpl.java index ab7255e..04015f1 100644 --- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionAggregateSourceImpl.java +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionAggregateSourceImpl.java @@ -18,23 +18,27 @@ package org.apache.hadoop.hbase.regionserver; -import java.util.TreeSet; +import java.util.HashSet; +import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.ReentrantReadWriteLock; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.metrics.BaseSourceImpl; import org.apache.hadoop.metrics2.MetricsCollector; import org.apache.hadoop.metrics2.MetricsRecordBuilder; +import org.apache.hadoop.metrics2.impl.JmxCacheBuster; +import org.apache.hadoop.metrics2.lib.Interns; +import org.apache.hadoop.metrics2.lib.MetricsExecutorImpl; @InterfaceAudience.Private public class MetricsRegionAggregateSourceImpl extends BaseSourceImpl implements MetricsRegionAggregateSource { - // lock to guard against concurrent access to regionSources private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); + private final MetricsExecutorImpl executor = new MetricsExecutorImpl(); - private final TreeSet regionSources = - new TreeSet(); + private final HashSet regionSources = + new HashSet(); public MetricsRegionAggregateSourceImpl() { this(METRICS_NAME, METRICS_DESCRIPTION, METRICS_CONTEXT, METRICS_JMX_CONTEXT); @@ -46,13 +50,21 @@ public class MetricsRegionAggregateSourceImpl extends BaseSourceImpl String metricsContext, String metricsJmxContext) { super(metricsName, metricsDescription, metricsContext, metricsJmxContext); + + // Every few mins clean the JMX cache. + executor.getExecutor().scheduleWithFixedDelay(new Runnable() { + public void run() { + JmxCacheBuster.clearJmxCache(true); + } + }, 5, 5, TimeUnit.MINUTES); } @Override public void register(MetricsRegionSource source) { lock.writeLock().lock(); try { - regionSources.add((MetricsRegionSourceImpl) source); + regionSources.add(source); + clearCache(); } finally { lock.writeLock().unlock(); } @@ -63,11 +75,23 @@ public class MetricsRegionAggregateSourceImpl extends BaseSourceImpl lock.writeLock().lock(); try { regionSources.remove(source); + clearCache(); } finally { lock.writeLock().unlock(); } } + private synchronized void clearCache() { + JmxCacheBuster.clearJmxCache(true); + executor.getExecutor().schedule(new Runnable() { + @Override + public void run() { + JmxCacheBuster.clearJmxCache(); + } + }, 5, TimeUnit.MINUTES); + + } + /** * Yes this is a get function that doesn't return anything. Thanks Hadoop for breaking all * expectations of java programmers. Instead of returning anything Hadoop metrics expects @@ -78,21 +102,21 @@ public class MetricsRegionAggregateSourceImpl extends BaseSourceImpl */ @Override public void getMetrics(MetricsCollector collector, boolean all) { - - MetricsRecordBuilder mrb = collector.addRecord(metricsName); if (regionSources != null) { lock.readLock().lock(); try { - for (MetricsRegionSourceImpl regionMetricSource : regionSources) { - regionMetricSource.snapshot(mrb, all); + for (MetricsRegionSource regionMetricSource : regionSources) { + if (regionMetricSource instanceof MetricsRegionSourceImpl) { + ((MetricsRegionSourceImpl) regionMetricSource).snapshot(mrb, all); + } } } finally { lock.readLock().unlock(); } + mrb.addGauge(Interns.info(NUM_REGIONS, NUMBER_OF_REGIONS_DESC), regionSources.size()); + metricsRegistry.snapshot(mrb, all); } - - metricsRegistry.snapshot(mrb, all); } } diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSourceImpl.java index df23942..714b199 100644 --- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSourceImpl.java +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSourceImpl.java @@ -26,6 +26,8 @@ import org.apache.commons.math.stat.descriptive.DescriptiveStatistics; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.metrics2.MetricsRecordBuilder; import org.apache.hadoop.metrics2.impl.JmxCacheBuster; +import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; +import org.apache.hadoop.metrics2.lib.DefaultMetricsSystemHelper; import org.apache.hadoop.metrics2.lib.DynamicMetricsRegistry; import org.apache.hadoop.metrics2.lib.Interns; import org.apache.hadoop.metrics2.lib.MutableCounterLong; @@ -34,13 +36,13 @@ import org.apache.hadoop.metrics2.lib.MutableHistogram; @InterfaceAudience.Private public class MetricsRegionSourceImpl implements MetricsRegionSource { - private final MetricsRegionWrapper regionWrapper; + private static final Log LOG = LogFactory.getLog(MetricsRegionSourceImpl.class); private boolean closed = false; + private MetricsRegionWrapper regionWrapper; private MetricsRegionAggregateSourceImpl agg; private DynamicMetricsRegistry registry; - private static final Log LOG = LogFactory.getLog(MetricsRegionSourceImpl.class); private String regionNamePrefix; private String regionPutKey; @@ -100,17 +102,18 @@ public class MetricsRegionSourceImpl implements MetricsRegionSource { closed = true; agg.deregister(this); - LOG.trace("Removing region Metrics: " + regionWrapper.getRegionName()); + if (LOG.isTraceEnabled()) { + LOG.trace("Removing region Metrics: " + regionWrapper.getRegionName()); + } + registry.removeMetric(regionPutKey); registry.removeMetric(regionDeleteKey); - registry.removeMetric(regionIncrementKey); - registry.removeMetric(regionAppendKey); + registry.removeHistogramMetrics(regionGetKey); + registry.removeHistogramMetrics(regionScanNextKey); - registry.removeMetric(regionGetKey); - registry.removeMetric(regionScanNextKey); - + this.regionWrapper = null; JmxCacheBuster.clearJmxCache(); } @@ -162,14 +165,13 @@ public class MetricsRegionSourceImpl implements MetricsRegionSource { @Override public int hashCode() { - return this.regionWrapper.getRegionName().hashCode(); + return regionWrapper.getRegionHashCode(); } @Override public boolean equals(Object obj) { if (obj == this) return true; - if (!(obj instanceof MetricsRegionSourceImpl)) return false; - return compareTo((MetricsRegionSourceImpl)obj) == 0; + return obj instanceof MetricsRegionSourceImpl && compareTo((MetricsRegionSourceImpl) obj) == 0; } void snapshot(MetricsRecordBuilder mrb, boolean ignored) { @@ -186,17 +188,24 @@ public class MetricsRegionSourceImpl implements MetricsRegionSource { MetricsRegionServerSource.MEMSTORE_SIZE_DESC), this.regionWrapper.getMemstoreSize()); mrb.addGauge(Interns.info(regionNamePrefix + MetricsRegionServerSource.STOREFILE_SIZE, - MetricsRegionServerSource.STOREFILE_SIZE_DESC), + MetricsRegionServerSource.STOREFILE_SIZE_DESC), this.regionWrapper.getStoreFileSize()); mrb.addCounter(Interns.info(regionNamePrefix + MetricsRegionSource.COMPACTIONS_COMPLETED_COUNT, - MetricsRegionSource.COMPACTIONS_COMPLETED_DESC), + MetricsRegionSource.COMPACTIONS_COMPLETED_DESC), this.regionWrapper.getNumCompactionsCompleted()); mrb.addCounter(Interns.info(regionNamePrefix + MetricsRegionSource.NUM_BYTES_COMPACTED_COUNT, - MetricsRegionSource.NUM_BYTES_COMPACTED_DESC), + MetricsRegionSource.NUM_BYTES_COMPACTED_DESC), this.regionWrapper.getNumBytesCompacted()); mrb.addCounter(Interns.info(regionNamePrefix + MetricsRegionSource.NUM_FILES_COMPACTED_COUNT, MetricsRegionSource.NUM_FILES_COMPACTED_DESC), this.regionWrapper.getNumFilesCompacted()); + mrb.addCounter(Interns.info(regionNamePrefix + MetricsRegionServerSource.READ_REQUEST_COUNT, + MetricsRegionServerSource.READ_REQUEST_COUNT_DESC), + this.regionWrapper.getReadRequestCount()); + mrb.addCounter(Interns.info(regionNamePrefix + MetricsRegionServerSource.WRITE_REQUEST_COUNT, + MetricsRegionServerSource.WRITE_REQUEST_COUNT_DESC), + this.regionWrapper.getWriteRequestCount()); + for (Map.Entry entry : this.regionWrapper .getCoprocessorExecutionStatistics() .entrySet()) { diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/impl/JmxCacheBuster.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/impl/JmxCacheBuster.java index 5d83150..52d706f 100644 --- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/impl/JmxCacheBuster.java +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/impl/JmxCacheBuster.java @@ -20,6 +20,7 @@ package org.apache.hadoop.metrics2.impl; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -36,33 +37,38 @@ import org.apache.hadoop.metrics2.lib.MetricsExecutorImpl; * This class need to be in the o.a.h.metrics2.impl namespace as many of the variables/calls used * are package private. */ -@edu.umd.cs.findbugs.annotations.SuppressWarnings( - value="LI_LAZY_INIT_STATIC", - justification="Yeah, its weird but its what we want") @InterfaceAudience.Private public class JmxCacheBuster { private static final Log LOG = LogFactory.getLog(JmxCacheBuster.class); - private static Object lock = new Object(); - private static ScheduledFuture fut = null; + private static AtomicReference fut = new AtomicReference<>(null); private static MetricsExecutor executor = new MetricsExecutorImpl(); /** * For JMX to forget about all previously exported metrics. */ + public static void clearJmxCache() { + clearJmxCache(false); + } + public static synchronized void clearJmxCache(boolean force) { //If there are more then 100 ms before the executor will run then everything should be merged. - synchronized (lock) { - if (fut == null || (!fut.isDone() && fut.getDelay(TimeUnit.MILLISECONDS) > 100)) return; - fut = executor.getExecutor().schedule(new JmxCacheBusterRunnable(), 5, TimeUnit.SECONDS); + ScheduledFuture future = fut.get(); + if (!force && + (future == null || (!future.isDone() && future.getDelay(TimeUnit.MILLISECONDS) > 100))) { + // BAIL OUT + return; } + future = executor.getExecutor().schedule(new JmxCacheBusterRunnable(), 5, TimeUnit.SECONDS); + fut.set(future); } static class JmxCacheBusterRunnable implements Runnable { - @Override public void run() { - LOG.trace("Clearing JMX mbean cache."); + if (LOG.isTraceEnabled()) { + LOG.trace("Clearing JMX mbean cache."); + } // This is pretty extreme but it's the best way that // I could find to get metrics to be removed. @@ -71,9 +77,9 @@ public class JmxCacheBuster { DefaultMetricsSystem.instance().stop(); DefaultMetricsSystem.instance().start(); } - } catch (Exception exception ) { - LOG.debug("error clearing the jmx it appears the metrics system hasn't been started", exception); + LOG.debug("error clearing the jmx it appears the metrics system hasn't been started", + exception); } } } diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/DefaultMetricsSystemHelper.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/DefaultMetricsSystemHelper.java new file mode 100644 index 0000000..832e220 --- /dev/null +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/DefaultMetricsSystemHelper.java @@ -0,0 +1,55 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.metrics2.lib; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +import java.lang.reflect.Method; + +public class DefaultMetricsSystemHelper { + + private static final Log LOG = LogFactory.getLog(DefaultMetricsSystemHelper.class); + private final Method removeObjectMethod; + + public DefaultMetricsSystemHelper() { + Method m; + try { + Class clazz = DefaultMetricsSystem.INSTANCE.getClass(); + m = clazz.getDeclaredMethod("removeObjectName", String.class); + m.setAccessible(true); + } catch (NoSuchMethodException e) { + m = null; + } + removeObjectMethod = m; + } + + public boolean removeObjectName(final String name) { + if (removeObjectMethod != null) { + try { + removeObjectMethod.invoke(DefaultMetricsSystem.INSTANCE, name); + return true; + } catch (Exception e) { + if (LOG.isTraceEnabled()) { + LOG.trace("Unable to remove object name from cache: " + name, e); + } + } + } + return false; + } +} diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/DynamicMetricsRegistry.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/DynamicMetricsRegistry.java index 281200c..7986d71 100644 --- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/DynamicMetricsRegistry.java +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/DynamicMetricsRegistry.java @@ -18,9 +18,12 @@ package org.apache.hadoop.metrics2.lib; +import java.lang.reflect.Method; import java.util.Collection; import java.util.concurrent.ConcurrentMap; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.metrics2.MetricsException; import org.apache.hadoop.metrics2.MetricsInfo; @@ -44,18 +47,22 @@ import com.google.common.collect.Maps; */ @InterfaceAudience.Private public class DynamicMetricsRegistry { + private static final Log LOG = LogFactory.getLog(DynamicMetricsRegistry.class); + private final ConcurrentMap metricsMap = Maps.newConcurrentMap(); private final ConcurrentMap tagsMap = Maps.newConcurrentMap(); private final MetricsInfo metricsInfo; + private final DefaultMetricsSystemHelper helper = new DefaultMetricsSystemHelper(); + private final static String[] histogramSuffixes = new String[]{"_num_ops", "_min", "_max", "_median", "_75th_percentile", "_95th_percentile", "_99th_percentile"}; /** * Construct the registry with a record name * @param name of the record of the metrics */ public DynamicMetricsRegistry(String name) { - metricsInfo = Interns.info(name, name); + this(Interns.info(name,name)); } /** @@ -405,9 +412,16 @@ public class DynamicMetricsRegistry { * @param name name of the metric to remove */ public void removeMetric(String name) { + helper.removeObjectName(name); metricsMap.remove(name); } + public void removeHistogramMetrics(String baseName) { + for (String suffix:histogramSuffixes) { + removeMetric(baseName+suffix); + } + } + /** * Get a MetricMutableGaugeLong from the storage. If it is not there atomically put it. * @@ -540,6 +554,9 @@ public class DynamicMetricsRegistry { } public void clearMetrics() { + for (String name:metricsMap.keySet()) { + helper.removeObjectName(name); + } metricsMap.clear(); } } diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MetricsExecutorImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MetricsExecutorImpl.java index 18a759e..897084e 100644 --- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MetricsExecutorImpl.java +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MetricsExecutorImpl.java @@ -27,7 +27,9 @@ import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.metrics2.MetricsExecutor; /** - * Class to handle the ScheduledExecutorService{@link ScheduledExecutorService} used by MetricMutableQuantiles{@link MetricMutableQuantiles} + * Class to handle the ScheduledExecutorService{@link ScheduledExecutorService} used by + * MetricMutableQuantiles{@link MetricMutableQuantiles}, MetricsRegionAggregateSourceImpl, and + * JmxCacheBuster */ @InterfaceAudience.Private public class MetricsExecutorImpl implements MetricsExecutor { @@ -46,8 +48,8 @@ public class MetricsExecutorImpl implements MetricsExecutor { private enum ExecutorSingleton { INSTANCE; - - private final ScheduledExecutorService scheduler = new ScheduledThreadPoolExecutor(1, new ThreadPoolExecutorThreadFactory("HBase-Metrics2-")); + private final ScheduledExecutorService scheduler = new ScheduledThreadPoolExecutor(1, + new ThreadPoolExecutorThreadFactory("HBase-Metrics2-")); } private static class ThreadPoolExecutorThreadFactory implements ThreadFactory { diff --git a/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionSourceImpl.java b/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionSourceImpl.java index 4be8905..28b1d62 100644 --- a/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionSourceImpl.java +++ b/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionSourceImpl.java @@ -125,6 +125,11 @@ public class TestMetricsRegionSourceImpl { } @Override + public int getRegionHashCode() { + return regionName.hashCode(); + } + + @Override public Map getCoprocessorExecutionStatistics() { return null; } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperImpl.java index be05f3e..78df787 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperImpl.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperImpl.java @@ -135,6 +135,11 @@ public class MetricsRegionWrapperImpl implements MetricsRegionWrapper, Closeable return this.region.compactionsFinished.get(); } + @Override + public int getRegionHashCode() { + return this.region.hashCode(); + } + public class HRegionMetricsWrapperRunnable implements Runnable { @Override diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperStub.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperStub.java index 2658c0a..94ac356 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperStub.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperStub.java @@ -86,6 +86,11 @@ public class MetricsRegionWrapperStub implements MetricsRegionWrapper { } @Override + public int getRegionHashCode() { + return 42; + } + + @Override public Map getCoprocessorExecutionStatistics() { return new HashMap(); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRemoveRegionMetrics.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRemoveRegionMetrics.java new file mode 100644 index 0000000..df973a5 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRemoveRegionMetrics.java @@ -0,0 +1,131 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.CompatibilityFactory; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.MiniHBaseCluster; +import org.apache.hadoop.hbase.NamespaceDescriptor; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.RegionLocator; +import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.test.MetricsAssertHelper; +import org.apache.hadoop.hbase.testclassification.LargeTests; + +import org.apache.hadoop.hbase.testclassification.RegionServerTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Threads; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import java.io.IOException; + +@Category({RegionServerTests.class, LargeTests.class}) +public class TestRemoveRegionMetrics { + + private static MiniHBaseCluster cluster; + private static Configuration conf; + private static HBaseTestingUtility TEST_UTIL; + private static MetricsAssertHelper metricsHelper; + + @BeforeClass + public static void startCluster() throws Exception { + metricsHelper = CompatibilityFactory.getInstance(MetricsAssertHelper.class); + TEST_UTIL = new HBaseTestingUtility(); + conf = TEST_UTIL.getConfiguration(); + conf.getLong("hbase.splitlog.max.resubmit", 0); + // Make the failure test faster + conf.setInt("zookeeper.recovery.retry", 0); + conf.setInt(HConstants.REGIONSERVER_INFO_PORT, -1); + + TEST_UTIL.startMiniCluster(1, 2); + cluster = TEST_UTIL.getHBaseCluster(); + + cluster.waitForActiveAndReadyMaster(); + + while (cluster.getLiveRegionServerThreads().size() < 2) { + Threads.sleep(100); + } + } + + + @Test + public void testMoveRegion() throws IOException, InterruptedException { + String tableNameString = "testMoveRegion"; + TableName tableName = TableName.valueOf(tableNameString); + Table t = TEST_UTIL.createTable(tableName, Bytes.toBytes("D")); + TEST_UTIL.waitUntilAllRegionsAssigned(t.getName()); + HBaseAdmin admin = TEST_UTIL.getHBaseAdmin(); + HRegionInfo regionInfo; + byte[] row = Bytes.toBytes("r1"); + + + for (int i = 0; i < 30; i++) { + boolean moved = false; + try (RegionLocator locator = TEST_UTIL.getConnection().getRegionLocator(tableName)) { + regionInfo = locator.getRegionLocation(row, true).getRegionInfo(); + } + + int currentServerIdx = cluster.getServerWith(regionInfo.getRegionName()); + int destServerIdx = (currentServerIdx +1)% cluster.getLiveRegionServerThreads().size(); + HRegionServer currentServer = cluster.getRegionServer(currentServerIdx); + HRegionServer destServer = cluster.getRegionServer(destServerIdx); + byte[] destServerName = Bytes.toBytes(destServer.getServerName().getServerName()); + + + // Do a put. The counters should be non-zero now + Put p = new Put(row); + p.addColumn(Bytes.toBytes("D"), Bytes.toBytes("Zero"), Bytes.toBytes("VALUE")); + t.put(p); + + + MetricsRegionAggregateSource currentAgg = currentServer.getRegion(regionInfo.getRegionName()).getMetrics().getSource().getAggregateSource(); + + String prefix = "namespace_"+ NamespaceDescriptor.DEFAULT_NAMESPACE_NAME_STR+ + "_table_"+tableNameString + + "_region_" + regionInfo.getEncodedName()+ + "_metric"; + + metricsHelper.assertCounter(prefix + "_mutateCount", 1, currentAgg); + + + try { + admin.move(regionInfo.getEncodedNameAsBytes(), destServerName); + moved = true; + Thread.sleep(5000); + } catch (IOException ioe) { + moved = false; + } + TEST_UTIL.waitUntilAllRegionsAssigned(t.getName()); + + if (moved) { + MetricsRegionAggregateSource destAgg = destServer.getRegion(regionInfo.getRegionName()).getMetrics().getSource().getAggregateSource(); + metricsHelper.assertCounter(prefix + "_mutateCount", 0, destAgg); + } + } + + TEST_UTIL.deleteTable(tableName); + + } +} -- 2.4.3