diff --git hbase-assembly/pom.xml hbase-assembly/pom.xml index b9d8dcc..19bf5f0 100644 --- hbase-assembly/pom.xml +++ hbase-assembly/pom.xml @@ -208,6 +208,16 @@ org.apache.hbase + hbase-metrics-api + ${project.version} + + + org.apache.hbase + hbase-metrics + ${project.version} + + + org.apache.hbase hbase-resource-bundle ${project.version} true diff --git hbase-examples/src/main/java/org/apache/hadoop/hbase/coprocessor/example/ExampleMasterObserverWithMetrics.java hbase-examples/src/main/java/org/apache/hadoop/hbase/coprocessor/example/ExampleMasterObserverWithMetrics.java new file mode 100644 index 0000000..a5bae0a --- /dev/null +++ hbase-examples/src/main/java/org/apache/hadoop/hbase/coprocessor/example/ExampleMasterObserverWithMetrics.java @@ -0,0 +1,142 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.coprocessor.example; + +import java.io.IOException; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.CoprocessorEnvironment; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.coprocessor.BaseMasterObserver; +import org.apache.hadoop.hbase.coprocessor.MasterCoprocessorEnvironment; +import org.apache.hadoop.hbase.coprocessor.ObserverContext; +import org.apache.hadoop.hbase.metrics.Counter; +import org.apache.hadoop.hbase.metrics.Gauge; +import org.apache.hadoop.hbase.metrics.MetricRegistry; +import org.apache.hadoop.hbase.metrics.Timer; + +/** + * An example coprocessor that collects some metrics to demonstrate the usage of exporting custom + * metrics from the coprocessor. + * + *

+ * These metrics will be available through the regular Hadoop metrics2 sinks (ganglia, opentsdb, + * etc) as well as JMX output. You can view a snapshot of the metrics by going to the http web UI + * of the master page, something like http://mymasterhost:16010/jmx + *

+ * @see ExampleRegionObserverWithMetrics + */ +public class ExampleMasterObserverWithMetrics extends BaseMasterObserver { + + private static final Log LOG = LogFactory.getLog(ExampleMasterObserverWithMetrics.class); + + /** This is the Timer metric object to keep track of the current count across invocations */ + private Timer createTableTimer; + private long createTableStartTime = Long.MIN_VALUE; + + /** This is a Counter object to keep track of disableTable operations */ + private Counter disableTableCounter; + + /** Returns the total memory of the process. We will use this to define a gauge metric */ + private long getTotalMemory() { + return Runtime.getRuntime().totalMemory(); + } + + /** Returns the max memory of the process. We will use this to define a gauge metric */ + private long getMaxMemory() { + return Runtime.getRuntime().maxMemory(); + } + + @Override + public void preCreateTable(ObserverContext ctx, + HTableDescriptor desc, HRegionInfo[] regions) throws IOException { + super.preCreateTable(ctx, desc, regions); + // we rely on the fact that there is only 1 instance of our MasterObserver. We keep track of + // when the operation starts before the operation is executing. + this.createTableStartTime = System.currentTimeMillis(); + } + + @Override + public void postCreateTable(ObserverContext ctx, + HTableDescriptor desc, HRegionInfo[] regions) throws IOException { + super.postCreateTable(ctx, desc, regions); + if (this.createTableStartTime > 0) { + long time = System.currentTimeMillis() - this.createTableStartTime; + LOG.info("Create table took: " + time); + + // Update the timer metric for the create table operation duration. + createTableTimer.updateMillis(time); + } + } + + @Override + public void preDisableTable(ObserverContext ctx, TableName tableName) throws IOException { + super.preDisableTable(ctx, tableName); + + // Increment the Counter for disable table operations + this.disableTableCounter.increment(); + } + + @Override + public void start(CoprocessorEnvironment env) throws IOException { + super.start(env); + + // start for the MasterObserver will be called only once in the lifetime of the + // server. We will construct and register all metrics that we will track across method + // invocations. + + if (env instanceof MasterCoprocessorEnvironment) { + // Obtain the MetricRegistry for the Master. Metrics from this registry will be reported + // at the master level per-server. + MetricRegistry registry = + ((MasterCoprocessorEnvironment) env).getMetricRegistryForMaster(); + + if (createTableTimer == null) { + // Create a new Counter, or get the already registered counter. + // It is much better to only call this once and save the Counter as a class field instead + // of creating the counter every time a coprocessor method is invoked. This will negate + // any performance bottleneck coming from map lookups tracking metrics in the registry. + createTableTimer = registry.timer("CreateTable"); + + // on stop(), we can remove these registered metrics via calling registry.remove(). But + // it is not needed for coprocessors at the master level. If coprocessor is stopped, + // the server is stopping anyway, so there will not be any resource leaks. + } + + if (disableTableCounter == null) { + disableTableCounter = registry.counter("DisableTable"); + } + + // Register a custom gauge. The Gauge object will be registered in the metrics registry and + // periodically the getValue() is invoked to obtain the snapshot. + registry.register("totalMemory", new Gauge() { + @Override + public Long getValue() { + return getTotalMemory(); + } + }); + + // Register a custom gauge using Java-8 lambdas (Supplier converted into Gauge) + registry.register("maxMemory", this::getMaxMemory); + } + } +} diff --git hbase-examples/src/main/java/org/apache/hadoop/hbase/coprocessor/example/ExampleRegionObserverWithMetrics.java hbase-examples/src/main/java/org/apache/hadoop/hbase/coprocessor/example/ExampleRegionObserverWithMetrics.java new file mode 100644 index 0000000..7606b05 --- /dev/null +++ hbase-examples/src/main/java/org/apache/hadoop/hbase/coprocessor/example/ExampleRegionObserverWithMetrics.java @@ -0,0 +1,121 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.apache.hadoop.hbase.coprocessor.example; + +import java.io.IOException; +import java.util.List; +import java.util.concurrent.ThreadLocalRandom; + +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.CoprocessorEnvironment; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver; +import org.apache.hadoop.hbase.coprocessor.ObserverContext; +import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment; +import org.apache.hadoop.hbase.metrics.Counter; +import org.apache.hadoop.hbase.metrics.MetricRegistry; +import org.apache.hadoop.hbase.metrics.Timer; + +/** + * An example coprocessor that collects some metrics to demonstrate the usage of exporting custom + * metrics from the coprocessor. + *

+ * These metrics will be available through the regular Hadoop metrics2 sinks (ganglia, opentsdb, + * etc) as well as JMX output. You can view a snapshot of the metrics by going to the http web UI + * of the regionserver page, something like http://myregionserverhost:16030/jmx + *

+ * + * @see ExampleMasterObserverWithMetrics + */ +public class ExampleRegionObserverWithMetrics extends BaseRegionObserver { + + private Counter preGetCounter; + private Timer costlyOperationTimer; + + @Override + public void preGetOp(ObserverContext e, Get get, List results) + throws IOException { + super.preGetOp(e, get, results); + + // Increment the Counter whenever the coprocessor is called + preGetCounter.increment(); + } + + @Override + public void postGetOp(ObserverContext e, Get get, + List results) throws IOException { + super.postGetOp(e, get, results); + + // do a costly (high latency) operation which we want to measure how long it takes by + // using a Timer (which is a Meter and a Histogram). + long start = System.nanoTime(); + try { + performCostlyOperation(); + } finally { + costlyOperationTimer.updateNanos(System.nanoTime() - start); + } + } + + private void performCostlyOperation() { + try { + // simulate the operation by sleeping. + Thread.sleep(ThreadLocalRandom.current().nextLong(100)); + } catch (InterruptedException ignore) {} + } + + @Override + public void start(CoprocessorEnvironment env) throws IOException { + super.start(env); + + // start for the RegionServerObserver will be called only once in the lifetime of the + // server. We will construct and register all metrics that we will track across method + // invocations. + + if (env instanceof RegionCoprocessorEnvironment) { + // Obtain the MetricRegistry for the RegionServer. Metrics from this registry will be reported + // at the region server level per-regionserver. + MetricRegistry registry = + ((RegionCoprocessorEnvironment) env).getMetricRegistryForRegionServer(); + + if (preGetCounter == null) { + // Create a new Counter, or get the already registered counter. + // It is much better to only call this once and save the Counter as a class field instead + // of creating the counter every time a coprocessor method is invoked. This will negate + // any performance bottleneck coming from map lookups tracking metrics in the registry. + // Returned counter instance is shared by all coprocessors of the same class in the same + // region server. + preGetCounter = registry.counter("preGetRequests"); + } + + if (costlyOperationTimer == null) { + // Create a Timer to track execution times for the costly operation. + costlyOperationTimer = registry.timer("costlyOperation"); + } + } + } + + @Override + public void stop(CoprocessorEnvironment e) throws IOException { + // we should NOT remove / deregister the metrics in stop(). The whole registry will be + // removed when the last region of the table is closed. + super.stop(e); + } +} diff --git hbase-hadoop-compat/pom.xml hbase-hadoop-compat/pom.xml index 016dd24..1ec1047 100644 --- hbase-hadoop-compat/pom.xml +++ hbase-hadoop-compat/pom.xml @@ -122,6 +122,10 @@ org.apache.commons commons-math
+ + org.apache.hbase + hbase-metrics-api + diff --git hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/metrics/BaseSource.java hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/metrics/BaseSource.java index f79aa9f..652aae1 100644 --- hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/metrics/BaseSource.java +++ hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/metrics/BaseSource.java @@ -104,4 +104,9 @@ public interface BaseSource { */ String getMetricsName(); + default MetricRegistryInfo getMetricRegistryInfo() { + return new MetricRegistryInfo(getMetricsName(), getMetricsDescription(), + getMetricsContext(), getMetricsJmxContext(), true); + } + } diff --git hbase-hadoop2-compat/pom.xml hbase-hadoop2-compat/pom.xml index 56c3e7f..3e63391 100644 --- hbase-hadoop2-compat/pom.xml +++ hbase-hadoop2-compat/pom.xml @@ -166,6 +166,10 @@ limitations under the License. org.apache.hbase + hbase-metrics + + + org.apache.hbase hbase-hadoop-compat ${project.version} test-jar diff --git hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/metrics/BaseSourceImpl.java hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/metrics/BaseSourceImpl.java index f843ec2..f57c745 100644 --- hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/metrics/BaseSourceImpl.java +++ hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/metrics/BaseSourceImpl.java @@ -19,6 +19,9 @@ package org.apache.hadoop.hbase.metrics; import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.metrics.impl.GlobalMetricRegistriesSource; +import org.apache.hadoop.hbase.metrics.impl.HBaseMetrics2HadoopMetricsAdapter; +import org.apache.hadoop.hbase.regionserver.MetricsRegionServerSourceImpl; import org.apache.hadoop.metrics2.MetricsCollector; import org.apache.hadoop.metrics2.MetricsSource; import org.apache.hadoop.metrics2.impl.JmxCacheBuster; @@ -47,15 +50,53 @@ public class BaseSourceImpl implements BaseSource, MetricsSource { inited = true; DefaultMetricsSystem.initialize(HBASE_METRICS_SYSTEM_NAME); JvmMetrics.initSingleton(name, ""); + // initialize hbase-metrics module based metric system as well. GlobalMetricRegistriesSource + // initialization depends on the metric system being already initialized, that is why we are + // doing it here. Once BaseSourceSourceImpl is removed, we should do the initialization of + // these elsewhere. + GlobalMetricRegistriesSource.init(); } } + /** + * @deprecated Use hbase-metrics/hbase-metrics-api module interfaces for new metrics. + * Defining BaseSources for new metric groups (WAL, RPC, etc) is not needed anymore, however, + * for existing BaseSource implemetnations, please use the field named "registry" which is a + * MetricRegistry instance together with the HBaseMetrics2HadoopMetricsAdapter. + */ + @Deprecated protected final DynamicMetricsRegistry metricsRegistry; protected final String metricsName; protected final String metricsDescription; protected final String metricsContext; protected final String metricsJmxContext; + /** + * Note that there are at least 4 MetricRegistry definitions in the source code. The first one is + * Hadoop Metrics2 MetricRegistry, second one is DynamicMetricsRegistry which is HBase's fork + * of the Hadoop metrics2 class. The third one is the dropwizard metrics implementation of + * MetricRegistry, and finally a new API abstraction in HBase that is the + * o.a.h.h.metrics.MetricRegistry class. This last one is the new way to use metrics within the + * HBase code. However, the others are in play because of existing metrics2 based code still + * needs to coexists until we get rid of all of our BaseSource and convert them to the new + * framework. Until that happens, new metrics can use the new API, but will be collected + * through the HBaseMetrics2HadoopMetricsAdapter class. + * + * BaseSourceImpl has two MetricRegistries. metricRegistry is for hadoop Metrics2 based + * metrics, while the registry is for hbase-metrics based metrics. + */ + protected final MetricRegistry registry; + + /** + * The adapter from hbase-metrics module to metrics2. This adepter is the connection between the + * Metrics in the MetricRegistry and the Hadoop Metrics2 system. Using this adapter, existing + * BaseSource implementations can define new metrics using the hbase-metrics/hbase-metrics-api + * module interfaces and still be able to make use of metrics2 sinks (including JMX). Existing + * BaseSources should call metricsAdapter.snapshotAllMetrics() in getMetrics() method. See + * {@link MetricsRegionServerSourceImpl}. + */ + protected final HBaseMetrics2HadoopMetricsAdapter metricsAdapter; + public BaseSourceImpl( String metricsName, String metricsDescription, @@ -72,6 +113,11 @@ public class BaseSourceImpl implements BaseSource, MetricsSource { //Register this instance. DefaultMetricsSystem.instance().register(metricsJmxContext, metricsDescription, this); + + // hbase-metrics module based metrics are registered in the hbase MetricsRegistry. + registry = MetricRegistries.global().create(this.getMetricRegistryInfo()); + metricsAdapter = new HBaseMetrics2HadoopMetricsAdapter(); + init(); } diff --git hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/metrics/impl/GlobalMetricRegistriesSource.java hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/metrics/impl/GlobalMetricRegistriesSource.java new file mode 100644 index 0000000..d955c28 --- /dev/null +++ hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/metrics/impl/GlobalMetricRegistriesSource.java @@ -0,0 +1,157 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.metrics.impl; + + +import java.util.Collection; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map.Entry; +import java.util.Optional; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.metrics.MetricRegistries; +import org.apache.hadoop.hbase.metrics.MetricRegistry; +import org.apache.hadoop.hbase.metrics.MetricRegistryInfo; +import org.apache.hadoop.metrics2.MetricsCollector; +import org.apache.hadoop.metrics2.MetricsSource; +import org.apache.hadoop.metrics2.impl.JmxCacheBuster; +import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; +import org.apache.hadoop.metrics2.lib.DefaultMetricsSystemHelper; + +/** + * This is a "dummy" MetricSource for all of the metrics in all of the MetricRegistry's in the + * MetricRegistries.global() instance. The source itself does not have any metrics of it's own, + * but acts as an adapter to export the MetricRegistry's in the global registry. Each MetricRegistry + * will be collected like a BaseSource instance for a group of metrics (like WAL, RPC, etc) with + * the MetricRegistryInfo's JMX context. + * + *

Developer note: + * This is not extending BaseSource or using BaseSourceImpl (but implements lower level + * MetricsSource) because unlike the current metrics2 based approach, the new metrics approach + * (hbase-metrics-api and hbase-metrics modules) work by having different MetricRegistries that are + * initialized and used from the code that lives in their respective modules (hbase-server, etc). + * There is no need to define BaseSource classes and do a lot of indirection. The MetricRegistry'es + * will be in the global MetricRegistriesImpl, and this class will iterate over + * MetricRegistries.global() and report the actual values using the metrics2 adaptor. + * + * We do not initialize the Hadoop Metrics2 system assuming that other BaseSources already do so + * (see BaseSourceImpl). Once the last BaseSource is moved to the new system, the metric2 + * initialization should be moved here. + *

+ */ +public class GlobalMetricRegistriesSource implements MetricsSource { + + private static final Log LOG = LogFactory.getLog(GlobalMetricRegistriesSource.class); + + private static final String NAME = "GlobalMetricRegistriesSource"; + private static final String DESCRIPTION + = "Dummy MetricSource for dynamic metrics based on hbase-metrics module"; + + private class MetricsSourceAdapter implements MetricsSource { + private final MetricRegistry registry; + MetricsSourceAdapter(MetricRegistry registry) { + this.registry = registry; + } + + @Override + public void getMetrics(MetricsCollector collector, boolean all) { + metricsAdapter.snapshotAllMetrics(registry, collector); + } + } + + private final DefaultMetricsSystemHelper helper; + private final HBaseMetrics2HadoopMetricsAdapter metricsAdapter; + private final HashMap registeredSources; + + private GlobalMetricRegistriesSource() { + this.metricsAdapter = new HBaseMetrics2HadoopMetricsAdapter(); + this.registeredSources = new HashMap<>(); + helper = new DefaultMetricsSystemHelper(); + } + + /** + * Make sure that this global MetricSource for hbase-metrics module based metrics are initialized + * and registered in the Hadoop metrics 2 system. + */ + public static GlobalMetricRegistriesSource init() { + GlobalMetricRegistriesSource source = new GlobalMetricRegistriesSource(); + DefaultMetricsSystem.instance().register(NAME, DESCRIPTION, source); + return source; + } + + @Override + public synchronized void getMetrics(MetricsCollector collector, boolean all) { + if (LOG.isTraceEnabled()) { + LOG.trace("getMetrics called: " + registeredSources); + } + + Collection registries = MetricRegistries.global().getMetricRegistries(); + for (MetricRegistry registry : registries) { + MetricRegistryInfo info = registry.getMetricRegistryInfo(); + + if (info.isExistingSource()) { + // If there is an already existing BaseSource for this MetricRegistry, skip it here. These + // types of registries are there only due to existing BaseSource implementations in the + // source code (like MetricsRegionServer, etc). This is to make sure that we can transition + // iteratively to the new hbase-metrics system. These type of MetricRegistry metrics will be + // exported from the BaseSource.getMetrics() call directly because there is already a + // MetricRecordBuilder there (see MetricsRegionServerSourceImpl). + continue; + } + + if (!registeredSources.containsKey(info)) { + if (LOG.isDebugEnabled()) { + LOG.debug("Registering adapter for the MetricRegistry: " + info.getMetricsJmxContext()); + } + // register this as a MetricSource under different JMX Context'es. + MetricsSourceAdapter adapter = new MetricsSourceAdapter(registry); + DefaultMetricsSystem.instance().register(info.getMetricsJmxContext(), + info.getMetricsDescription(), adapter); + registeredSources.put(info, adapter); + // next collection will collect the newly registered MetricSource. Doing this here leads to + // ConcurrentModificationException. + } + } + + boolean removed = false; + // Remove registered sources if it is removed from the global registry + for (Iterator> it = + registeredSources.entrySet().iterator(); it.hasNext();) { + Entry entry = it.next(); + MetricRegistryInfo info = entry.getKey(); + Optional found = MetricRegistries.global().get(info); + if (!found.isPresent()) { + if (LOG.isDebugEnabled()) { + LOG.debug("Removing adapter for the MetricRegistry: " + info.getMetricsJmxContext()); + } + DefaultMetricsSystem.instance().unregisterSource(info.getMetricsJmxContext()); + helper.removeSourceName(info.getMetricsJmxContext()); + helper.removeObjectName(info.getMetricsJmxContext()); + it.remove(); + removed = true; + } + } + if (removed) { + JmxCacheBuster.clearJmxCache(); + } + } + + +} diff --git hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/metrics/impl/HBaseMetrics2HadoopMetricsAdapter.java hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/metrics/impl/HBaseMetrics2HadoopMetricsAdapter.java new file mode 100644 index 0000000..ec4a1a7 --- /dev/null +++ hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/metrics/impl/HBaseMetrics2HadoopMetricsAdapter.java @@ -0,0 +1,169 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/* + * Copyright 2016 Josh Elser + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.metrics.impl; + +import java.util.Map; + +import org.apache.commons.lang.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.metrics.Counter; +import org.apache.hadoop.hbase.metrics.Gauge; +import org.apache.hadoop.hbase.metrics.Histogram; +import org.apache.hadoop.hbase.metrics.Meter; +import org.apache.hadoop.hbase.metrics.Metric; +import org.apache.hadoop.hbase.metrics.MetricRegistry; +import org.apache.hadoop.hbase.metrics.MetricRegistryInfo; +import org.apache.hadoop.hbase.metrics.Timer; +import org.apache.hadoop.metrics2.MetricsCollector; +import org.apache.hadoop.metrics2.MetricsInfo; +import org.apache.hadoop.metrics2.MetricsRecordBuilder; +import org.apache.hadoop.metrics2.lib.Interns; +import org.apache.hadoop.metrics2.lib.MutableHistogram; + +/** + * This is the adapter from "HBase Metrics Framework", implemented in hbase-metrics-api and + * hbase-metrics modules to the Hadoop Metrics2 framework. This adapter is not a metric source, + * but a helper to be able to collect all of the Metric's in the MetricRegistry using the + * MetricsCollector and MetricsRecordBuilder. + * + * Some of the code is forked from https://github.com/joshelser/dropwizard-hadoop-metrics2. + */ +public class HBaseMetrics2HadoopMetricsAdapter { + private static final Log LOG + = LogFactory.getLog(HBaseMetrics2HadoopMetricsAdapter.class); + private static final String EMPTY_STRING = ""; + + public HBaseMetrics2HadoopMetricsAdapter() { + } + + /** + * Iterates over the MetricRegistry and adds them to the {@code collector}. + * + * @param collector A metrics collector + */ + public void snapshotAllMetrics(MetricRegistry metricRegistry, + MetricsCollector collector) { + MetricRegistryInfo info = metricRegistry.getMetricRegistryInfo(); + MetricsRecordBuilder builder = collector.addRecord(Interns.info(info.getMetricsName(), + info.getMetricsDescription())); + builder.setContext(info.getMetricsContext()); + + snapshotAllMetrics(metricRegistry, builder); + } + + /** + * Iterates over the MetricRegistry and adds them to the {@code builder}. + * + * @param builder A record builder + */ + public void snapshotAllMetrics(MetricRegistry metricRegistry, MetricsRecordBuilder builder) { + Map metrics = metricRegistry.getMetrics(); + + for (Map.Entry e: metrics.entrySet()) { + // Always capitalize the name + String name = StringUtils.capitalize(e.getKey()); + Metric metric = e.getValue(); + + if (metric instanceof Gauge) { + addGauge(name, (Gauge) metric, builder); + } else if (metric instanceof Counter) { + addCounter(name, (Counter)metric, builder); + } else if (metric instanceof Histogram) { + addHistogram(name, (Histogram)metric, builder); + } else if (metric instanceof Meter) { + addMeter(name, (Meter)metric, builder); + } else if (metric instanceof Timer) { + addTimer(name, (Timer)metric, builder); + } else { + LOG.info("Ignoring unknown Metric class " + metric.getClass().getName()); + } + } + } + + private void addGauge(String name, Gauge gauge, MetricsRecordBuilder builder) { + final MetricsInfo info = Interns.info(name, EMPTY_STRING); + final Object o = gauge.getValue(); + + // Figure out which gauge types metrics2 supports and call the right method + if (o instanceof Integer) { + builder.addGauge(info, (int) o); + } else if (o instanceof Long) { + builder.addGauge(info, (long) o); + } else if (o instanceof Float) { + builder.addGauge(info, (float) o); + } else if (o instanceof Double) { + builder.addGauge(info, (double) o); + } else { + LOG.warn("Ignoring Gauge (" + name + ") with unhandled type: " + o.getClass()); + } + } + + private void addCounter(String name, Counter counter, MetricsRecordBuilder builder) { + MetricsInfo info = Interns.info(name, EMPTY_STRING); + builder.addCounter(info, counter.getCount()); + } + + /** + * Add Histogram value-distribution data to a Hadoop-Metrics2 record building. + * + * @param builder A Hadoop-Metrics2 record builder. + * @param name A base name for this record. + * @param desc A description for this record. + * @param snapshot The distribution of measured values. + */ + private void addHistogram(String name, Histogram histogram, MetricsRecordBuilder builder) { + MutableHistogram.snapshot(name, EMPTY_STRING, histogram, builder, true); + } + + /** + * Add Dropwizard-Metrics rate information to a Hadoop-Metrics2 record builder, converting the + * rates to the appropriate unit. + * + * @param builder A Hadoop-Metrics2 record builder. + * @param name A base name for this record. + */ + private void addMeter(String name, Meter meter, MetricsRecordBuilder builder) { + builder.addGauge(Interns.info(name + "_count", EMPTY_STRING), meter.getCount()); + builder.addGauge(Interns.info(name + "_mean_rate", EMPTY_STRING), meter.getMeanRate()); + builder.addGauge(Interns.info(name + "_1min_rate", EMPTY_STRING), meter.getOneMinuteRate()); + builder.addGauge(Interns.info(name + "_5min_rate", EMPTY_STRING), meter.getFiveMinuteRate()); + builder.addGauge(Interns.info(name + "_15min_rate", EMPTY_STRING), + meter.getFifteenMinuteRate()); + } + + private void addTimer(String name, Timer timer, MetricsRecordBuilder builder) { + addMeter(name, timer.getMeter(), builder); + addHistogram(name, timer.getHistogram(), builder); + } +} diff --git hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java index b412fd1..67764f8 100644 --- hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java +++ hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java @@ -510,6 +510,12 @@ public class MetricsRegionServerSourceImpl } metricsRegistry.snapshot(mrb, all); + + // source is registered in supers constructor, sometimes called before the whole initialization. + if (metricsAdapter != null) { + // snapshot MetricRegistry as well + metricsAdapter.snapshotAllMetrics(registry, mrb); + } } @Override diff --git hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/DefaultMetricsSystemHelper.java hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/DefaultMetricsSystemHelper.java index 832e220..9d1e499 100644 --- hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/DefaultMetricsSystemHelper.java +++ hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/DefaultMetricsSystemHelper.java @@ -17,26 +17,44 @@ */ package org.apache.hadoop.metrics2.lib; +import java.lang.reflect.Field; +import java.lang.reflect.Method; +import java.util.HashMap; + import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import java.lang.reflect.Method; - public class DefaultMetricsSystemHelper { private static final Log LOG = LogFactory.getLog(DefaultMetricsSystemHelper.class); private final Method removeObjectMethod; + private final Field sourceNamesField; + private final Field mapField; public DefaultMetricsSystemHelper() { + Class clazz = DefaultMetricsSystem.INSTANCE.getClass(); Method m; try { - Class clazz = DefaultMetricsSystem.INSTANCE.getClass(); m = clazz.getDeclaredMethod("removeObjectName", String.class); m.setAccessible(true); } catch (NoSuchMethodException e) { m = null; } removeObjectMethod = m; + + Field f1, f2; + try { + f1 = clazz.getDeclaredField("sourceNames"); + f1.setAccessible(true); + f2 = UniqueNames.class.getDeclaredField("map"); + f2.setAccessible(true); + } catch (NoSuchFieldException e) { + LOG.trace(e); + f1 = null; + f2 = null; + } + sourceNamesField = f1; + mapField = f2; } public boolean removeObjectName(final String name) { @@ -52,4 +70,29 @@ public class DefaultMetricsSystemHelper { } return false; } + + /** + * Unfortunately Hadoop tries to be too-clever and permanently keeps track of all names registered + * so far as a Source, thus preventing further re-registration of the source with the same name. + * In case of dynamic metrics tied to region-lifecycles, this becomes a problem because we would + * like to be able to re-register and remove with the same name. Otherwise, it is resource leak. + * This ugly code manually removes the name from the UniqueNames map. + */ + public void removeSourceName(String name) { + if (sourceNamesField == null || mapField == null) { + return; + } + try { + Object sourceNames = sourceNamesField.get(DefaultMetricsSystem.INSTANCE); + HashMap map = (HashMap) mapField.get(sourceNames); + synchronized (sourceNames) { + map.remove(name); + } + } catch (Exception ex) { + if (LOG.isTraceEnabled()) { + LOG.trace("Received exception while trying to access Hadoop Metrics classes via reflection.", + ex); + } + } + } } diff --git hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MutableHistogram.java hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MutableHistogram.java index ec8cb50..6a2f203 100644 --- hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MutableHistogram.java +++ hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MutableHistogram.java @@ -18,12 +18,12 @@ package org.apache.hadoop.metrics2.lib; -import java.util.concurrent.atomic.LongAdder; - import org.apache.commons.lang.StringUtils; import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.metrics.Histogram; import org.apache.hadoop.hbase.metrics.Interns; -import org.apache.hadoop.hbase.util.FastLongHistogram; +import org.apache.hadoop.hbase.metrics.Snapshot; +import org.apache.hadoop.hbase.metrics.impl.HistogramImpl; import org.apache.hadoop.metrics2.MetricHistogram; import org.apache.hadoop.metrics2.MetricsInfo; import org.apache.hadoop.metrics2.MetricsRecordBuilder; @@ -33,28 +33,9 @@ import org.apache.hadoop.metrics2.MetricsRecordBuilder; */ @InterfaceAudience.Private public class MutableHistogram extends MutableMetric implements MetricHistogram { - // Double buffer the two FastLongHistograms. - // As they are reset they learn how the buckets should be spaced - // So keep two around and use them - protected final FastLongHistogram histogram; - + protected HistogramImpl histogram; protected final String name; protected final String desc; - protected final LongAdder counter = new LongAdder(); - - private boolean metricsInfoStringInited = false; - private String NUM_OPS_METRIC; - private String MIN_METRIC; - private String MAX_METRIC; - private String MEAN_METRIC; - private String MEDIAN_METRIC; - private String TWENTY_FIFTH_PERCENTILE_METRIC; - private String SEVENTY_FIFTH_PERCENTILE_METRIC; - private String NINETIETH_PERCENTILE_METRIC; - private String NINETY_FIFTH_PERCENTILE_METRIC; - private String NINETY_EIGHTH_PERCENTILE_METRIC; - private String NINETY_NINETH_PERCENTILE_METRIC; - private String NINETY_NINE_POINT_NINETH_PERCENTILE_METRIC; public MutableHistogram(MetricsInfo info) { this(info.name(), info.description()); @@ -67,12 +48,11 @@ public class MutableHistogram extends MutableMetric implements MetricHistogram { protected MutableHistogram(String name, String description, long maxExpected) { this.name = StringUtils.capitalize(name); this.desc = StringUtils.uncapitalize(description); - this.histogram = new FastLongHistogram(FastLongHistogram.DEFAULT_NBINS, 1, maxExpected); + this.histogram = new HistogramImpl(); } public void add(final long val) { - counter.increment(); - histogram.add(val, 1); + histogram.update(val); } public long getMax() { @@ -81,56 +61,42 @@ public class MutableHistogram extends MutableMetric implements MetricHistogram { @Override public synchronized void snapshot(MetricsRecordBuilder metricsRecordBuilder, boolean all) { - // Get a reference to the old histogram. - FastLongHistogram histo = histogram.reset(); - if (histo != null) { - updateSnapshotMetrics(metricsRecordBuilder, histo); - } + snapshot(name, desc, histogram, metricsRecordBuilder, all); } - protected void updateSnapshotMetrics(MetricsRecordBuilder metricsRecordBuilder, - FastLongHistogram histo) { - if (!metricsInfoStringInited) { - NUM_OPS_METRIC = name + NUM_OPS_METRIC_NAME; - MIN_METRIC = name + MIN_METRIC_NAME; - MAX_METRIC = name + MAX_METRIC_NAME; - MEAN_METRIC = name + MEAN_METRIC_NAME; - MEDIAN_METRIC = name + MEDIAN_METRIC_NAME; - TWENTY_FIFTH_PERCENTILE_METRIC = name + TWENTY_FIFTH_PERCENTILE_METRIC_NAME; - SEVENTY_FIFTH_PERCENTILE_METRIC = name + SEVENTY_FIFTH_PERCENTILE_METRIC_NAME; - NINETIETH_PERCENTILE_METRIC = name + NINETIETH_PERCENTILE_METRIC_NAME; - NINETY_FIFTH_PERCENTILE_METRIC = name + NINETY_FIFTH_PERCENTILE_METRIC_NAME; - NINETY_EIGHTH_PERCENTILE_METRIC = name + NINETY_EIGHTH_PERCENTILE_METRIC_NAME; - NINETY_NINETH_PERCENTILE_METRIC = name + NINETY_NINETH_PERCENTILE_METRIC_NAME; - NINETY_NINE_POINT_NINETH_PERCENTILE_METRIC = name + - NINETY_NINE_POINT_NINETH_PERCENTILE_METRIC_NAME; - - metricsInfoStringInited = true; + public static void snapshot(String name, String desc, Histogram histogram, + MetricsRecordBuilder metricsRecordBuilder, boolean all) { + // Get a reference to the old histogram. + Snapshot snapshot = histogram.snapshot(); + if (snapshot != null) { + updateSnapshotMetrics(name, desc, histogram, snapshot, metricsRecordBuilder); } + } - metricsRecordBuilder.addCounter(Interns.info(NUM_OPS_METRIC, desc), counter.sum()); - metricsRecordBuilder.addGauge(Interns.info(MIN_METRIC, desc), histo.getMin()); - metricsRecordBuilder.addGauge(Interns.info(MAX_METRIC, desc), histo.getMax()); - metricsRecordBuilder.addGauge(Interns.info(MEAN_METRIC, desc), histo.getMean()); - - long[] percentiles = histo.getQuantiles(); - - metricsRecordBuilder.addGauge(Interns.info(TWENTY_FIFTH_PERCENTILE_METRIC, desc), - percentiles[0]); - metricsRecordBuilder.addGauge(Interns.info(MEDIAN_METRIC, desc), - percentiles[1]); - metricsRecordBuilder.addGauge(Interns.info(SEVENTY_FIFTH_PERCENTILE_METRIC, desc), - percentiles[2]); - metricsRecordBuilder.addGauge(Interns.info(NINETIETH_PERCENTILE_METRIC, desc), - percentiles[3]); - metricsRecordBuilder.addGauge(Interns.info(NINETY_FIFTH_PERCENTILE_METRIC, desc), - percentiles[4]); - metricsRecordBuilder.addGauge(Interns.info(NINETY_EIGHTH_PERCENTILE_METRIC, desc), - percentiles[5]); - metricsRecordBuilder.addGauge(Interns.info(NINETY_NINETH_PERCENTILE_METRIC, desc), - percentiles[6]); + protected static void updateSnapshotMetrics(String name, String desc, Histogram histogram, + Snapshot snapshot, MetricsRecordBuilder metricsRecordBuilder) { + metricsRecordBuilder.addCounter(Interns.info(name + NUM_OPS_METRIC_NAME, desc), + histogram.getCount()); + metricsRecordBuilder.addGauge(Interns.info(name + MIN_METRIC_NAME, desc), snapshot.getMin()); + metricsRecordBuilder.addGauge(Interns.info(name + MAX_METRIC_NAME, desc), snapshot.getMax()); + metricsRecordBuilder.addGauge(Interns.info(name + MEAN_METRIC_NAME, desc), snapshot.getMean()); + + metricsRecordBuilder.addGauge(Interns.info(name + TWENTY_FIFTH_PERCENTILE_METRIC_NAME, desc), + snapshot.get25thPercentile()); + metricsRecordBuilder.addGauge(Interns.info(name + MEDIAN_METRIC_NAME, desc), + snapshot.getMedian()); + metricsRecordBuilder.addGauge(Interns.info(name + SEVENTY_FIFTH_PERCENTILE_METRIC_NAME, desc), + snapshot.get75thPercentile()); + metricsRecordBuilder.addGauge(Interns.info(name + NINETIETH_PERCENTILE_METRIC_NAME, desc), + snapshot.get90thPercentile()); + metricsRecordBuilder.addGauge(Interns.info(name + NINETY_FIFTH_PERCENTILE_METRIC_NAME, desc), + snapshot.get95thPercentile()); + metricsRecordBuilder.addGauge(Interns.info(name + NINETY_EIGHTH_PERCENTILE_METRIC_NAME, desc), + snapshot.get98thPercentile()); + metricsRecordBuilder.addGauge(Interns.info(name + NINETY_NINETH_PERCENTILE_METRIC_NAME, desc), + snapshot.get99thPercentile()); metricsRecordBuilder.addGauge( - Interns.info(NINETY_NINE_POINT_NINETH_PERCENTILE_METRIC, desc), - percentiles[7]); + Interns.info(name + NINETY_NINE_POINT_NINETH_PERCENTILE_METRIC_NAME, desc), + snapshot.get999thPercentile()); } } diff --git hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MutableRangeHistogram.java hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MutableRangeHistogram.java index 94bcdaa..a12dc27 100644 --- hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MutableRangeHistogram.java +++ hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MutableRangeHistogram.java @@ -20,7 +20,7 @@ package org.apache.hadoop.metrics2.lib; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.metrics.Interns; -import org.apache.hadoop.hbase.util.FastLongHistogram; +import org.apache.hadoop.hbase.metrics.Snapshot; import org.apache.hadoop.metrics2.MetricHistogram; import org.apache.hadoop.metrics2.MetricsInfo; import org.apache.hadoop.metrics2.MetricsRecordBuilder; @@ -56,22 +56,22 @@ public abstract class MutableRangeHistogram extends MutableHistogram implements @Override public synchronized void snapshot(MetricsRecordBuilder metricsRecordBuilder, boolean all) { // Get a reference to the old histogram. - FastLongHistogram histo = histogram.reset(); - if (histo != null) { - updateSnapshotMetrics(metricsRecordBuilder, histo); - updateSnapshotRangeMetrics(metricsRecordBuilder, histo); + Snapshot snapshot = histogram.snapshot(); + if (snapshot != null) { + updateSnapshotMetrics(name, desc, histogram, snapshot, metricsRecordBuilder); + updateSnapshotRangeMetrics(metricsRecordBuilder, snapshot); } } public void updateSnapshotRangeMetrics(MetricsRecordBuilder metricsRecordBuilder, - FastLongHistogram histogram) { + Snapshot snapshot) { long priorRange = 0; long cumNum = 0; final long[] ranges = getRanges(); final String rangeType = getRangeType(); for (int i = 0; i < ranges.length - 1; i++) { - long val = histogram.getNumAtOrBelow(ranges[i]); + long val = snapshot.getCountAtOrBelow(ranges[i]); if (val - cumNum > 0) { metricsRecordBuilder.addCounter( Interns.info(name + "_" + rangeType + "_" + priorRange + "-" + ranges[i], desc), diff --git hbase-metrics-api/README.txt hbase-metrics-api/README.txt new file mode 100644 index 0000000..dfaa29f --- /dev/null +++ hbase-metrics-api/README.txt @@ -0,0 +1,78 @@ +Overview +======== +hbase-metrics and hbase-metrics-api are two modules that define and implement the "new" metric +system used internally within HBase. These two modules (and some other code in hbase-hadoop2-compat) +module are referred as "HBase metrics framework". + +HBase-metrics-api Module +======================== +HBase Metrics API (hbase-metrics-api) contains the interface +that HBase exposes internally and to third party code (including coprocessors). It is a thin +abstraction over the actual implementation for backwards compatibility guarantees. The source +/ binary and other compatibility guarantees are for "LimitedPrivate API" (see [1] for an +explanation). + +The metrics API in this hbase-metrics-api module is inspired by the Dropwizard metrics 3.1 API +(See [2]). It is a subset of the API only containing metrics collection. However, the implementation +is HBase-specific and provided in hbase-metrics module. All of the classes in this module is +HBase-internal. See the latest documentation of Dropwizard metrics for examples of defining / using +metrics. + + +HBase-metrics Module +==================== +hbase-metrics module contains implementation of the "HBase Metrics API", including MetricRegistry, +Counter, Histogram, etc. These are highly concurrent implementations of the Metric interfaces. +Metrics in HBase are grouped into different sets (like WAL, RPC, RegionServer, etc). Each group of +metrics should be tracked via a MetricRegistry specific to that group. Metrics can be dynamically +added or removed from the registry with a name. Each Registry is independent of the other +registries and will have it's own JMX context and MetricRecord (when used with Metrics2). + + +MetricRegistry's themselves are tracked via a global registry (of MetricRegistries) called +MetricRegistries. MetricRegistries.global() can be used to obtain the global instance. +MetricRegistry instances can also be dynamically registered and removed. However, unlike the +MetricRegistry, MetricRegistries does reference counting of the MetricRegistry instances. Only +Metrics in the MetricRegistry instances that are in the global MetricRegistry are exported to the +metric sinks or JMX. + + +Coprocessor Metrics +=================== +HBase allows custom coprocessors to track and export metrics using the new framework. +Coprocessors and other third party code should only use the classes and interfaces from +hbase-metrics-api module and only the classes that are marked with InterfaceAudience.LimitedPrivate +annotation. There is no guarantee on the compatibility requirements for other classes. + +Coprocessors can obtain the MetricRegistry to register their custom metrics via corresponding +CoprocessorEnvironment context. See ExampleRegionObserverWithMetrics and +ExampleMasterObserverWithMetrics classes in hbase-examples module for usage. + + +Developer Notes +=============== +Historically, HBase has been using Hadoop's Metrics2 framework [3] for collecting and reporting the +metrics internally. However, due to the difficultly of dealing with the Metrics2 framework, HBase is +moving away from Hadoop's metrics implementation to its custom implementation. The move will happen +incrementally, and during the time, both Hadoop Metrics2-based metrics and hbase-metrics module +based classes will be in the source code. All new implementations for metrics SHOULD use the new +API and framework. + +Examples of the new framework can be found in MetricsCoprocessor and MetricsRegionServerSourceImpl +classes. See HBASE-9774 [4] for more context. + +hbase-metrics module right now only deals with metrics tracking and collection. It does not do JMX +reporting or reporting to console, ganglia, opentsdb, etc. We use Hadoop's Metrics2 for reporting +metrics to different sinks or exporting via JMX. However, this is contained within the +hbase-hadoop2-compat module completely, so that rest of the code does not know anything about the +Metrics2 dependency. HBaseMetrics2HadoopMetricsAdapter is the adapter that can collect metrics +in a MetricRegistry using the metric2 MetricsCollector / MetricRecordBuilder interfaces. +GlobalMetricRegistriesSource is the global Metrics2 Source that collects all of the metrics in all +of the metric registries in the MetricRegistries.global() instance. + + +References +1. https://hbase.apache.org/book.html#hbase.versioning +2. http://metrics.dropwizard.io/ +3. https://hadoop.apache.org/docs/r2.7.2/api/org/apache/hadoop/metrics2/package-summary.html +4. https://issues.apache.org/jira/browse/HBASE-9774 \ No newline at end of file diff --git hbase-metrics-api/pom.xml hbase-metrics-api/pom.xml new file mode 100644 index 0000000..5822971 --- /dev/null +++ hbase-metrics-api/pom.xml @@ -0,0 +1,112 @@ + + + + 4.0.0 + + hbase + org.apache.hbase + 2.0.0-SNAPSHOT + .. + + + hbase-metrics-api + Apache HBase - Metrics API + HBase Metrics API descriptions + + + + + org.apache.maven.plugins + maven-site-plugin + + true + + + + + org.apache.maven.plugins + maven-source-plugin + + + + maven-assembly-plugin + ${maven.assembly.version} + + true + + + + maven-surefire-plugin + + + + secondPartTestsExecution + test + + test + + + true + + + + + + + + + + + org.apache.hbase + hbase-annotations + + + org.apache.hbase + hbase-annotations + test-jar + test + + + org.apache.hbase + hbase-common + + + + commons-logging + commons-logging + + + + + + + skip-metrics-api-tests + + + skip-metrics-api-tests + + + + true + + + + diff --git hbase-metrics-api/src/main/java/org/apache/hadoop/hbase/metrics/Counter.java hbase-metrics-api/src/main/java/org/apache/hadoop/hbase/metrics/Counter.java new file mode 100644 index 0000000..2e4147e --- /dev/null +++ hbase-metrics-api/src/main/java/org/apache/hadoop/hbase/metrics/Counter.java @@ -0,0 +1,60 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.metrics; + +import org.apache.hadoop.hbase.HBaseInterfaceAudience; +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.classification.InterfaceStability; + +/** + * A mutable number optimized for high concurrency counting. + */ +@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.COPROC) +@InterfaceStability.Evolving +public interface Counter extends Metric { + + /** + * Increment {@code this} by 1. + */ + void increment(); + + /** + * Increment {@code this} by {@code n}. + * + * @param n The amount to increment. + */ + void increment(long n); + + /** + * Decrement {@code this} by 1. + */ + void decrement(); + + /** + * Decrement {@code this} by {@code n}. + * + * @param n The amount to decrement. + */ + void decrement(long n); + + /** + * Returns the current value. + * @return the current value. + */ + long getCount(); +} \ No newline at end of file diff --git hbase-metrics-api/src/main/java/org/apache/hadoop/hbase/metrics/Gauge.java hbase-metrics-api/src/main/java/org/apache/hadoop/hbase/metrics/Gauge.java new file mode 100644 index 0000000..90df8e0 --- /dev/null +++ hbase-metrics-api/src/main/java/org/apache/hadoop/hbase/metrics/Gauge.java @@ -0,0 +1,35 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.metrics; + +import org.apache.hadoop.hbase.HBaseInterfaceAudience; +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.classification.InterfaceStability; + +/** + * A metrics which measures a discrete value. + * + * @param The value of the Gauge. + */ +@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.COPROC) +@InterfaceStability.Evolving +public interface Gauge extends Metric { + + T getValue(); + +} \ No newline at end of file diff --git hbase-metrics-api/src/main/java/org/apache/hadoop/hbase/metrics/Histogram.java hbase-metrics-api/src/main/java/org/apache/hadoop/hbase/metrics/Histogram.java new file mode 100644 index 0000000..b5b54c7 --- /dev/null +++ hbase-metrics-api/src/main/java/org/apache/hadoop/hbase/metrics/Histogram.java @@ -0,0 +1,58 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.metrics; + +import org.apache.hadoop.hbase.HBaseInterfaceAudience; +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.classification.InterfaceStability; + +/** + * A metric which measures the distribution of values. + */ +@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.COPROC) +@InterfaceStability.Evolving +public interface Histogram extends Metric { + + /** + * Adds a new value to the distribution. + * + * @param value The value to add + */ + void update(int value); + + /** + * Adds a new value to the distribution. + * + * @param value The value to add + */ + void update(long value); + + /** + * Return the total number of values added to the histogram. + * @return the total number of values. + */ + long getCount(); + + /** + * Snapshot the current values in the Histogram + * @return a Snapshot of the distribution. + */ + @InterfaceAudience.Private + Snapshot snapshot(); + +} diff --git hbase-metrics-api/src/main/java/org/apache/hadoop/hbase/metrics/Meter.java hbase-metrics-api/src/main/java/org/apache/hadoop/hbase/metrics/Meter.java new file mode 100644 index 0000000..fccaa38 --- /dev/null +++ hbase-metrics-api/src/main/java/org/apache/hadoop/hbase/metrics/Meter.java @@ -0,0 +1,90 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.metrics; + +import org.apache.hadoop.hbase.HBaseInterfaceAudience; +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.classification.InterfaceStability; + +/** + * A metric which measure the rate at which some operation is invoked. + */ +@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.COPROC) +@InterfaceStability.Evolving +public interface Meter extends Metric { + + /** + * Records one occurrence. + */ + void mark(); + + /** + * Records {@code events} occurrences. + * + * @param events Number of occurrences to record. + */ + void mark(long events); + + /** + * Returns the number of events. + * @return the number of events. + */ + long getCount(); + + /** + * Returns the mean rate at which events have occurred since the meter was created. + * @return the mean rate at which events have occurred since the meter was created + */ + double getMeanRate(); + + /** + * Returns the one-minute exponentially-weighted moving average rate at which events have + * occurred since the meter was created. + *

+ * This rate has the same exponential decay factor as the one-minute load average in the {@code + * top} Unix command. + * + * @return the one-minute exponentially-weighted moving average rate at which events have + * occurred since the meter was created + */ + double getOneMinuteRate(); + + /** + * Returns the five-minute exponentially-weighted moving average rate at which events have + * occurred since the meter was created. + *

+ * This rate has the same exponential decay factor as the five-minute load average in the {@code + * top} Unix command. + * + * @return the five-minute exponentially-weighted moving average rate at which events have + * occurred since the meter was created + */ + double getFiveMinuteRate(); + + /** + * Returns the fifteen-minute exponentially-weighted moving average rate at which events have + * occurred since the meter was created. + *

+ * This rate has the same exponential decay factor as the fifteen-minute load average in the + * {@code top} Unix command. + * + * @return the fifteen-minute exponentially-weighted moving average rate at which events have + * occurred since the meter was created + */ + double getFifteenMinuteRate(); +} \ No newline at end of file diff --git hbase-metrics-api/src/main/java/org/apache/hadoop/hbase/metrics/Metric.java hbase-metrics-api/src/main/java/org/apache/hadoop/hbase/metrics/Metric.java new file mode 100644 index 0000000..0a31803 --- /dev/null +++ hbase-metrics-api/src/main/java/org/apache/hadoop/hbase/metrics/Metric.java @@ -0,0 +1,30 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.metrics; + +import org.apache.hadoop.hbase.HBaseInterfaceAudience; +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.classification.InterfaceStability; + +/** + * Parent interface for all metrics. + */ +@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.COPROC) +@InterfaceStability.Evolving +public interface Metric { +} diff --git hbase-metrics-api/src/main/java/org/apache/hadoop/hbase/metrics/MetricRegistries.java hbase-metrics-api/src/main/java/org/apache/hadoop/hbase/metrics/MetricRegistries.java new file mode 100644 index 0000000..502119d --- /dev/null +++ hbase-metrics-api/src/main/java/org/apache/hadoop/hbase/metrics/MetricRegistries.java @@ -0,0 +1,89 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.metrics; + +import java.util.Collection; +import java.util.Optional; +import java.util.Set; + +import org.apache.hadoop.hbase.HBaseInterfaceAudience; +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.classification.InterfaceStability; + +/** + * MetricRegistries is collection of MetricRegistry's. MetricsRegistries implementations should do + * ref-counting of MetricRegistry's via create() and remove() methods. + */ +@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.COPROC) +@InterfaceStability.Evolving +public abstract class MetricRegistries { + + private static final class LazyHolder { + private static final MetricRegistries GLOBAL = MetricRegistriesLoader.load(); + } + + /** + * Return the global singleton instance for the MetricRegistries. + * @return MetricRegistries implementation. + */ + public static MetricRegistries global() { + return LazyHolder.GLOBAL; + } + + /** + * Removes all the MetricRegisties. + */ + public abstract void clear(); + + /** + * Create or return MetricRegistry with the given info. MetricRegistry will only be created + * if current reference count is 0. Otherwise ref counted is incremented, and an existing instance + * will be returned. + * @param info the info object for the MetricRegistrytry. + * @return created or existing MetricRegistry. + */ + public abstract MetricRegistry create(MetricRegistryInfo info); + + /** + * Decrements the ref count of the MetricRegistry, and removes if ref count == 0. + * @param key the info object for the MetricRegistrytry. + * @return true if metric registry is removed. + */ + public abstract boolean remove(MetricRegistryInfo key); + + /** + * Returns the MetricRegistry if found. + * @param info the info for the registry. + * @return a MetricRegistry optional. + */ + public abstract Optional get(MetricRegistryInfo info); + + /** + * Returns MetricRegistryInfo's for the MetricRegistry's created. + * @return MetricRegistryInfo's for the MetricRegistry's created. + */ + public abstract Set getMetricRegistryInfos(); + + /** + * Returns MetricRegistry's created. + * @return MetricRegistry's created. + */ + public abstract Collection getMetricRegistries(); +} diff --git hbase-metrics-api/src/main/java/org/apache/hadoop/hbase/metrics/MetricRegistriesLoader.java hbase-metrics-api/src/main/java/org/apache/hadoop/hbase/metrics/MetricRegistriesLoader.java new file mode 100644 index 0000000..4fef10c --- /dev/null +++ hbase-metrics-api/src/main/java/org/apache/hadoop/hbase/metrics/MetricRegistriesLoader.java @@ -0,0 +1,96 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.apache.hadoop.hbase.metrics; + + +import java.util.ArrayList; +import java.util.List; +import java.util.ServiceLoader; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.util.ReflectionUtils; + +import com.google.common.annotations.VisibleForTesting; + +@InterfaceAudience.Private +public class MetricRegistriesLoader { + private static final Log LOG = LogFactory.getLog(MetricRegistries.class); + + private static final String defaultClass + = "org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl"; + + /** + * Creates a {@link MetricRegistries} instance using the corresponding {@link MetricRegistries} + * available to {@link ServiceLoader} on the classpath. If no instance is found, then default + * implementation will be loaded. + * @return A {@link MetricRegistries} implementation. + */ + public static MetricRegistries load() { + List availableImplementations = getDefinedImplemantations(); + return load(availableImplementations); + } + + /** + * Creates a {@link MetricRegistries} instance using the corresponding {@link MetricRegistries} + * available to {@link ServiceLoader} on the classpath. If no instance is found, then default + * implementation will be loaded. + * @return A {@link MetricRegistries} implementation. + */ + @VisibleForTesting + static MetricRegistries load(List availableImplementations) { + + if (availableImplementations.size() == 1) { + // One and only one instance -- what we want/expect + MetricRegistries impl = availableImplementations.get(0); + LOG.info("Loaded MetricRegistries " + impl.getClass()); + return impl; + } else if (availableImplementations.isEmpty()) { + try { + return ReflectionUtils.newInstance((Class)Class.forName(defaultClass)); + } catch (ClassNotFoundException e) { + throw new RuntimeException(e); + } + } else { + // Tell the user they're doing something wrong, and choose the first impl. + StringBuilder sb = new StringBuilder(); + for (MetricRegistries factory : availableImplementations) { + if (sb.length() > 0) { + sb.append(", "); + } + sb.append(factory.getClass()); + } + LOG.warn("Found multiple MetricRegistries implementations: " + sb + + ". Using first found implementation: " + availableImplementations.get(0)); + return availableImplementations.get(0); + } + } + + private static List getDefinedImplemantations() { + ServiceLoader loader = ServiceLoader.load(MetricRegistries.class); + List availableFactories = new ArrayList<>(); + for (MetricRegistries impl : loader) { + availableFactories.add(impl); + } + return availableFactories; + } +} diff --git hbase-metrics-api/src/main/java/org/apache/hadoop/hbase/metrics/MetricRegistry.java hbase-metrics-api/src/main/java/org/apache/hadoop/hbase/metrics/MetricRegistry.java new file mode 100644 index 0000000..3138bca --- /dev/null +++ hbase-metrics-api/src/main/java/org/apache/hadoop/hbase/metrics/MetricRegistry.java @@ -0,0 +1,111 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.metrics; + +import java.util.Optional; + +import org.apache.hadoop.hbase.HBaseInterfaceAudience; +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.classification.InterfaceStability; + +/** + * General purpose factory for creating various metrics. + */ +@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.COPROC) +@InterfaceStability.Evolving +public interface MetricRegistry extends MetricSet { + + /** + * Get or construct a {@link Timer} used to measure durations and report rates. + * + * @param name the name of the timer. + * @return An instance of {@link Timer}. + */ + Timer timer(String name); + + /** + * Get or construct a {@link Histogram} used to measure a distribution of values. + * + * @param name The name of the Histogram. + * @return An instance of {@link Histogram}. + */ + Histogram histogram(String name); + + /** + * Get or construct a {@link Meter} used to measure durations and report distributions (a + * combination of a {@link Timer} and a {@link Histogram}. + * + * @param name The name of the Meter. + * @return An instance of {@link Meter}. + */ + Meter meter(String name); + + /** + * Get or construct a {@link Counter} used to track a mutable number. + * + * @param name The name of the Counter + * @return An instance of {@link Counter}. + */ + Counter counter(String name); + + /** + * Register a {@link Gauge}. The Gauge will be invoked at a period defined by the implementation + * of {@link MetricRegistry}. + * @param name The name of the Gauge. + * @param gauge A callback to compute the current value. + * @return the registered gauge, or the existing gauge + */ + Gauge register(String name, Gauge gauge); + + /** + * Registers the {@link Metric} with the given name if there does not exist one with the same + * name. Returns the newly registered or existing Metric. + * @param name The name of the Metric. + * @param metric the metric to register + * @return the registered metric, or the existing metrid + */ + Metric register(String name, Metric metric); + + /** + * Registers the {@link Metric}s in the given MetricSet. + * @param metricSet set of metrics to register. + */ + void registerAll(MetricSet metricSet); + + /** + * Returns previously registered metric with the name if any. + * @param name the name of the metric + * @return previously registered metric + */ + Optional get(String name); + + /** + * Removes the metric with the given name. + * + * @param name the name of the metric + * @return true if the metric is removed. + */ + boolean remove(String name); + + /** + * Return the MetricRegistryInfo object for this registry. + * @return MetricRegistryInfo describing the registry. + */ + @InterfaceAudience.Private + MetricRegistryInfo getMetricRegistryInfo(); +} \ No newline at end of file diff --git hbase-metrics-api/src/main/java/org/apache/hadoop/hbase/metrics/MetricRegistryFactory.java hbase-metrics-api/src/main/java/org/apache/hadoop/hbase/metrics/MetricRegistryFactory.java new file mode 100644 index 0000000..b161d20 --- /dev/null +++ hbase-metrics-api/src/main/java/org/apache/hadoop/hbase/metrics/MetricRegistryFactory.java @@ -0,0 +1,36 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.apache.hadoop.hbase.metrics; + +import org.apache.hadoop.hbase.classification.InterfaceAudience; + +/** + * A Factory for creating MetricRegistries. This is the main plugin point for metrics implementation + */ +@InterfaceAudience.Private +public interface MetricRegistryFactory { + /** + * Create a MetricRegistry from the given MetricRegistryInfo + * @param info the descriptor for MetricRegistry + * @return a MetricRegistry implementation + */ + MetricRegistry create(MetricRegistryInfo info); +} diff --git hbase-metrics-api/src/main/java/org/apache/hadoop/hbase/metrics/MetricRegistryInfo.java hbase-metrics-api/src/main/java/org/apache/hadoop/hbase/metrics/MetricRegistryInfo.java new file mode 100644 index 0000000..58fcba7 --- /dev/null +++ hbase-metrics-api/src/main/java/org/apache/hadoop/hbase/metrics/MetricRegistryInfo.java @@ -0,0 +1,112 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.metrics; + + +import org.apache.commons.lang.builder.HashCodeBuilder; +import org.apache.hadoop.hbase.classification.InterfaceAudience; + +/** + * HBase Metrics are grouped in different MetricRegistry'ies. All metrics that correspond to a + * subcomponent (like RPC, GC, WAL) are managed in a single MetricRegistry. + * This class holds the name and description and JMX related context names for such group of + * metrics. + */ +@InterfaceAudience.Private +public class MetricRegistryInfo { + + protected final String metricsName; + protected final String metricsDescription; + protected final String metricsContext; + protected final String metricsJmxContext; + protected final boolean existingSource; + + public MetricRegistryInfo( + String metricsName, + String metricsDescription, + String metricsJmxContext, + String metricsContext, + boolean existingSource) { + this.metricsName = metricsName; + this.metricsDescription = metricsDescription; + this.metricsContext = metricsContext; + this.metricsJmxContext = metricsJmxContext; + this.existingSource = existingSource; + } + + /** + * Get the metrics context. For hadoop metrics2 system this is usually an all lowercased string. + * eg. regionserver, master, thriftserver + * + * @return The string context used to register this source to hadoop's metrics2 system. + */ + public String getMetricsContext() { + return metricsContext; + } + + /** + * Get the description of what this source exposes. + */ + public String getMetricsDescription() { + return metricsDescription; + } + + /** + * Get the name of the context in JMX that this source will be exposed through. + * This is in ObjectName format. With the default context being Hadoop -> HBase + */ + public String getMetricsJmxContext() { + return metricsJmxContext; + } + + /** + * Get the name of the metrics that are being exported by this source. + * Eg. IPC, GC, WAL + */ + public String getMetricsName() { + return metricsName; + } + + /** + * Returns whether or not this MetricRegistry is for an existing BaseSource + * @return true if this MetricRegistry is for an existing BaseSource. + */ + public boolean isExistingSource() { + return existingSource; + } + + @Override + public boolean equals(Object obj) { + if (obj instanceof MetricRegistryInfo) { + return this.hashCode() == obj.hashCode(); + } else { + return false; + } + } + + @Override + public int hashCode() { + return new HashCodeBuilder() + .append(metricsName) + .append(metricsDescription) + .append(metricsContext) + .append(metricsJmxContext) + .hashCode(); + } +} diff --git hbase-metrics-api/src/main/java/org/apache/hadoop/hbase/metrics/MetricSet.java hbase-metrics-api/src/main/java/org/apache/hadoop/hbase/metrics/MetricSet.java new file mode 100644 index 0000000..9cf2378 --- /dev/null +++ hbase-metrics-api/src/main/java/org/apache/hadoop/hbase/metrics/MetricSet.java @@ -0,0 +1,41 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.metrics; + +import java.util.Map; + +import org.apache.hadoop.hbase.HBaseInterfaceAudience; +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.classification.InterfaceStability; + +/** + * A set of named metrics. + * + * @see MetricRegistry#registerAll(MetricSet) + */ +@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.COPROC) +@InterfaceStability.Evolving +public interface MetricSet extends Metric { + /** + * A map of metric names to metrics. + * + * @return the metrics + */ + Map getMetrics(); +} diff --git hbase-metrics-api/src/main/java/org/apache/hadoop/hbase/metrics/PackageMarker.java hbase-metrics-api/src/main/java/org/apache/hadoop/hbase/metrics/PackageMarker.java new file mode 100644 index 0000000..8278d0c --- /dev/null +++ hbase-metrics-api/src/main/java/org/apache/hadoop/hbase/metrics/PackageMarker.java @@ -0,0 +1,36 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.metrics; + +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; + +/** + * This is a dummy annotation that forces javac to produce output for + * otherwise empty package-info.java. + * + *

The result is maven-compiler-plugin can properly identify the scope of + * changed files + * + *

See more details in + * + * maven-compiler-plugin: incremental compilation broken + */ +@Retention(RetentionPolicy.SOURCE) +public @interface PackageMarker { +} diff --git hbase-metrics-api/src/main/java/org/apache/hadoop/hbase/metrics/Snapshot.java hbase-metrics-api/src/main/java/org/apache/hadoop/hbase/metrics/Snapshot.java new file mode 100644 index 0000000..56ee8ae --- /dev/null +++ hbase-metrics-api/src/main/java/org/apache/hadoop/hbase/metrics/Snapshot.java @@ -0,0 +1,136 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.apache.hadoop.hbase.metrics; + +import org.apache.hadoop.hbase.classification.InterfaceAudience; + +/** + * A statictical sample of histogram values. + */ +@InterfaceAudience.Private +public interface Snapshot { + + /** + * Return the values with the given quantiles. + * @param quantiles the requested quantiles. + * @return the value for the quantiles. + */ + long[] getQuantiles(double[] quantiles); + + /** + * Return the values with the default quantiles. + * @return the value for default the quantiles. + */ + long[] getQuantiles(); + + /** + * Returns the number of values in the snapshot. + * + * @return the number of values + */ + long getCount(); + + /** + * Returns the total count below the given value + * @param val the value + * @return the total count below the given value + */ + long getCountAtOrBelow(long val); + + /** + * Returns the value at the 25th percentile in the distribution. + * + * @return the value at the 25th percentile + */ + long get25thPercentile(); + + /** + * Returns the value at the 75th percentile in the distribution. + * + * @return the value at the 75th percentile + */ + long get75thPercentile(); + + /** + * Returns the value at the 90th percentile in the distribution. + * + * @return the value at the 90th percentile + */ + long get90thPercentile(); + + /** + * Returns the value at the 95th percentile in the distribution. + * + * @return the value at the 95th percentile + */ + long get95thPercentile(); + + /** + * Returns the value at the 98th percentile in the distribution. + * + * @return the value at the 98th percentile + */ + long get98thPercentile(); + + /** + * Returns the value at the 99th percentile in the distribution. + * + * @return the value at the 99th percentile + */ + long get99thPercentile(); + + /** + * Returns the value at the 99.9th percentile in the distribution. + * + * @return the value at the 99.9th percentile + */ + long get999thPercentile(); + + /** + * Returns the median value in the distribution. + * + * @return the median value + */ + long getMedian(); + + /** + * Returns the highest value in the snapshot. + * + * @return the highest value + */ + long getMax(); + + /** + * Returns the arithmetic mean of the values in the snapshot. + * + * @return the arithmetic mean + */ + long getMean(); + + /** + * Returns the lowest value in the snapshot. + * + * @return the lowest value + */ + long getMin(); + + // TODO: Dropwizard histograms also track stddev +} diff --git hbase-metrics-api/src/main/java/org/apache/hadoop/hbase/metrics/Timer.java hbase-metrics-api/src/main/java/org/apache/hadoop/hbase/metrics/Timer.java new file mode 100644 index 0000000..63fc649 --- /dev/null +++ hbase-metrics-api/src/main/java/org/apache/hadoop/hbase/metrics/Timer.java @@ -0,0 +1,68 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.metrics; + +import java.util.concurrent.TimeUnit; + +import org.apache.hadoop.hbase.HBaseInterfaceAudience; +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.classification.InterfaceStability; + +/** + * A metric which encompasses a {@link Histogram} and {@link Meter}. + */ +@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.COPROC) +@InterfaceStability.Evolving +public interface Timer extends Metric { + /** + * Update the timer with the given duration in given time unit. + * @param duration the duration of the event + * @param unit the time unit for the duration + */ + void update(long duration, TimeUnit unit); + + /** + * Update the timer with the given duration in milliseconds + * @param durationMillis the duration of the event in ms + */ + default void updateMillis(long durationMillis) { + update(durationMillis, TimeUnit.NANOSECONDS); + } + + /** + * Update the timer with the given duration in microseconds + * @param durationMicros the duration of the event in microseconds + */ + default void updateMicros(long durationMicros) { + update(durationMicros, TimeUnit.MICROSECONDS); + } + + /** + * Update the timer with the given duration in nanoseconds + * @param durationNanos the duration of the event in ns + */ + default void updateNanos(long durationNanos) { + update(durationNanos, TimeUnit.NANOSECONDS); + } + + @InterfaceAudience.Private + Histogram getHistogram(); + + @InterfaceAudience.Private + Meter getMeter(); +} \ No newline at end of file diff --git hbase-metrics-api/src/main/java/org/apache/hadoop/hbase/metrics/package-info.java hbase-metrics-api/src/main/java/org/apache/hadoop/hbase/metrics/package-info.java new file mode 100644 index 0000000..e79451f --- /dev/null +++ hbase-metrics-api/src/main/java/org/apache/hadoop/hbase/metrics/package-info.java @@ -0,0 +1,25 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Metrics API for HBase. + */ +@PackageMarker +package org.apache.hadoop.hbase.metrics; + +// End package-info.java diff --git hbase-metrics-api/src/test/java/org/apache/hadoop/hbase/metrics/TestMetricRegistriesLoader.java hbase-metrics-api/src/test/java/org/apache/hadoop/hbase/metrics/TestMetricRegistriesLoader.java new file mode 100644 index 0000000..8746146 --- /dev/null +++ hbase-metrics-api/src/test/java/org/apache/hadoop/hbase/metrics/TestMetricRegistriesLoader.java @@ -0,0 +1,56 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.metrics; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; +import static org.mockito.Mockito.mock; + +import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import com.google.common.collect.Lists; + +/** + * Test class for {@link MetricRegistriesLoader}. + */ +@Category(SmallTests.class) +public class TestMetricRegistriesLoader { + + @Test + public void testLoadSinleInstance() { + MetricRegistries loader = mock(MetricRegistries.class); + MetricRegistries instance = MetricRegistriesLoader.load(Lists.newArrayList(loader)); + assertEquals(loader, instance); + } + + @Test + public void testLoadMultipleInstances() { + MetricRegistries loader1 = mock(MetricRegistries.class); + MetricRegistries loader2 = mock(MetricRegistries.class); + MetricRegistries loader3 = mock(MetricRegistries.class); + MetricRegistries instance = MetricRegistriesLoader.load(Lists.newArrayList(loader1, loader2, + loader3)); + + // the load() returns the first instance + assertEquals(loader1, instance); + assertNotEquals(loader2, instance); + assertNotEquals(loader3, instance); + } +} \ No newline at end of file diff --git hbase-metrics/README.txt hbase-metrics/README.txt new file mode 100644 index 0000000..d80064c --- /dev/null +++ hbase-metrics/README.txt @@ -0,0 +1 @@ +See the documentation at hbase-metrics-api/README. \ No newline at end of file diff --git hbase-metrics/pom.xml hbase-metrics/pom.xml new file mode 100644 index 0000000..fbd2bd5 --- /dev/null +++ hbase-metrics/pom.xml @@ -0,0 +1,136 @@ + + + + 4.0.0 + + hbase + org.apache.hbase + 2.0.0-SNAPSHOT + .. + + + hbase-metrics + Apache HBase - Metrics Implementation + HBase Metrics Implementation + + + + + org.apache.maven.plugins + maven-site-plugin + + true + + + + + org.apache.maven.plugins + maven-source-plugin + + + + maven-assembly-plugin + ${maven.assembly.version} + + true + + + + maven-surefire-plugin + + + + secondPartTestsExecution + test + + test + + + true + + + + + + + + + + + org.apache.hbase + hbase-annotations + + + org.apache.hbase + hbase-annotations + test-jar + test + + + org.apache.hbase + hbase-common + + + org.apache.hbase + hbase-metrics-api + + + org.apache.hbase + hbase-metrics-api + test-jar + test + + + + commons-logging + commons-logging + + + io.dropwizard.metrics + metrics-core + + + junit + junit + test + + + org.mockito + mockito-all + test + + + + + + + skip-metrics-tests + + + skip-metrics-tests + + + + true + + + + diff --git hbase-metrics/src/main/java/org/apache/hadoop/hbase/metrics/impl/CounterImpl.java hbase-metrics/src/main/java/org/apache/hadoop/hbase/metrics/impl/CounterImpl.java new file mode 100644 index 0000000..147a903 --- /dev/null +++ hbase-metrics/src/main/java/org/apache/hadoop/hbase/metrics/impl/CounterImpl.java @@ -0,0 +1,61 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.metrics.impl; + +import java.util.concurrent.atomic.LongAdder; + +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.metrics.Counter; + +/** + * Custom implementation of {@link org.apache.hadoop.hbase.metrics.Counter} using LongAdder. + */ +@InterfaceAudience.Private +public class CounterImpl implements Counter { + + private final LongAdder counter; + + public CounterImpl() { + this.counter = new LongAdder(); + } + + @Override + public void increment() { + this.counter.increment(); + } + + @Override + public void increment(long n) { + this.counter.add(n); + } + + @Override + public void decrement() { + this.counter.decrement(); + } + + @Override + public void decrement(long n) { + this.counter.add(-n); + } + + @Override + public long getCount() { + return this.counter.sum(); + } +} \ No newline at end of file diff --git hbase-metrics/src/main/java/org/apache/hadoop/hbase/metrics/impl/DropwizardMeter.java hbase-metrics/src/main/java/org/apache/hadoop/hbase/metrics/impl/DropwizardMeter.java new file mode 100644 index 0000000..fc92483 --- /dev/null +++ hbase-metrics/src/main/java/org/apache/hadoop/hbase/metrics/impl/DropwizardMeter.java @@ -0,0 +1,74 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.metrics.impl; + +import java.util.Objects; + +import org.apache.hadoop.hbase.classification.InterfaceAudience; + +import com.codahale.metrics.Meter; + +/** + * Dropwizard metrics implementation of {@link org.apache.hadoop.hbase.metrics.Meter}. + */ +@InterfaceAudience.Private +public class DropwizardMeter implements org.apache.hadoop.hbase.metrics.Meter { + + private final Meter meter; + + public DropwizardMeter() { + this.meter = new Meter(); + } + + public DropwizardMeter(Meter meter) { + this.meter = Objects.requireNonNull(meter); + } + + @Override public void mark() { + this.meter.mark(); + } + + @Override public void mark(long count) { + this.meter.mark(count); + } + + @Override + public long getCount() { + return meter.getCount(); + } + + @Override + public double getMeanRate() { + return meter.getMeanRate(); + } + + @Override + public double getOneMinuteRate() { + return meter.getOneMinuteRate(); + } + + @Override + public double getFiveMinuteRate() { + return meter.getFiveMinuteRate(); + } + + @Override + public double getFifteenMinuteRate() { + return meter.getFifteenMinuteRate(); + } +} \ No newline at end of file diff --git hbase-common/src/main/java/org/apache/hadoop/hbase/util/FastLongHistogram.java hbase-metrics/src/main/java/org/apache/hadoop/hbase/metrics/impl/FastLongHistogram.java similarity index 80% rename from hbase-common/src/main/java/org/apache/hadoop/hbase/util/FastLongHistogram.java rename to hbase-metrics/src/main/java/org/apache/hadoop/hbase/metrics/impl/FastLongHistogram.java index 310348e..aa28193 100644 --- hbase-common/src/main/java/org/apache/hadoop/hbase/util/FastLongHistogram.java +++ hbase-metrics/src/main/java/org/apache/hadoop/hbase/metrics/impl/FastLongHistogram.java @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.hadoop.hbase.util; +package org.apache.hadoop.hbase.metrics.impl; import java.util.Arrays; import java.util.concurrent.atomic.AtomicLong; @@ -24,12 +24,14 @@ import java.util.stream.Stream; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; +import org.apache.hadoop.hbase.metrics.Snapshot; +import org.apache.hadoop.hbase.util.AtomicUtils; /** * FastLongHistogram is a thread-safe class that estimate distribution of data and computes the * quantiles. */ -@InterfaceAudience.Public +@InterfaceAudience.Private @InterfaceStability.Evolving public class FastLongHistogram { @@ -214,6 +216,20 @@ public class FastLongHistogram { long getNumAtOrBelow(long val) { return Arrays.stream(counts).mapToLong(c -> c.sum()).limit(getIndex(val) + 1).sum(); } + + public long getMin() { + long min = this.min.get(); + return min == Long.MAX_VALUE ? 0 : min; // in case it is not initialized + } + + public long getMean() { + long count = this.count.sum(); + long total = this.total.sum(); + if (count == 0) { + return 0; + } + return total / count; + } } // The bins counting values. It is replaced with a new one in calling of reset(). @@ -273,8 +289,7 @@ public class FastLongHistogram { } public long getMin() { - long min = this.bins.min.get(); - return min == Long.MAX_VALUE ? 0 : min; // in case it is not initialized + return this.bins.getMin(); } public long getMax() { @@ -286,13 +301,7 @@ public class FastLongHistogram { } public long getMean() { - Bins bins = this.bins; - long count = bins.count.sum(); - long total = bins.total.sum(); - if (count == 0) { - return 0; - } - return total / count; + return this.bins.getMean(); } public long getNumAtOrBelow(long value) { @@ -302,9 +311,87 @@ public class FastLongHistogram { /** * Resets the histogram for new counting. */ - public FastLongHistogram reset() { - Bins oldBins = this.bins; + public Snapshot snapshotAndReset() { + final Bins oldBins = this.bins; this.bins = new Bins(this.bins, this.bins.counts.length - 3, 0.01, 0.99); - return new FastLongHistogram(oldBins); + final long[] percentiles = oldBins.getQuantiles(DEFAULT_QUANTILES); + final long count = oldBins.count.sum(); + + return new Snapshot() { + @Override + public long[] getQuantiles(double[] quantiles) { + return oldBins.getQuantiles(quantiles); + } + + @Override + public long[] getQuantiles() { + return percentiles; + } + + @Override + public long getCount() { + return count; + } + + @Override + public long getCountAtOrBelow(long val) { + return oldBins.getNumAtOrBelow(val); + } + + @Override + public long get25thPercentile() { + return percentiles[0]; + } + + @Override + public long get75thPercentile() { + return percentiles[2]; + } + + @Override + public long get90thPercentile() { + return percentiles[3]; + } + + @Override + public long get95thPercentile() { + return percentiles[4]; + } + + @Override + public long get98thPercentile() { + return percentiles[5]; + } + + @Override + public long get99thPercentile() { + return percentiles[6]; + } + + @Override + public long get999thPercentile() { + return percentiles[7]; + } + + @Override + public long getMedian() { + return percentiles[1]; + } + + @Override + public long getMax() { + return oldBins.max.get(); + } + + @Override + public long getMean() { + return oldBins.getMean(); + } + + @Override + public long getMin() { + return oldBins.getMin(); + } + }; } } diff --git hbase-metrics/src/main/java/org/apache/hadoop/hbase/metrics/impl/HistogramImpl.java hbase-metrics/src/main/java/org/apache/hadoop/hbase/metrics/impl/HistogramImpl.java new file mode 100644 index 0000000..b52caf8 --- /dev/null +++ hbase-metrics/src/main/java/org/apache/hadoop/hbase/metrics/impl/HistogramImpl.java @@ -0,0 +1,81 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.apache.hadoop.hbase.metrics.impl; + +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.metrics.Histogram; +import org.apache.hadoop.hbase.metrics.Snapshot; + +/** + * Custom histogram implementation based on FastLongHistogram. Dropwizard-based histograms are + * slow compared to this implementation, so we are using our implementation here. + * See HBASE-15222. + */ +@InterfaceAudience.Private +public class HistogramImpl implements Histogram { + // Double buffer the two FastLongHistograms. + // As they are reset they learn how the buckets should be spaced + // So keep two around and use them + protected final FastLongHistogram histogram; + private final CounterImpl counter; + + public HistogramImpl() { + this(Integer.MAX_VALUE << 2); + } + + public HistogramImpl(long maxExpected) { + this(FastLongHistogram.DEFAULT_NBINS, 1, maxExpected); + } + + public HistogramImpl(int numBins, long min, long maxExpected) { + this.counter = new CounterImpl(); + this.histogram = new FastLongHistogram(numBins, min, maxExpected); + } + + protected HistogramImpl(CounterImpl counter, FastLongHistogram histogram) { + this.counter = counter; + this.histogram = histogram; + } + + @Override + public void update(int value) { + counter.increment(); + histogram.add(value, 1); + } + + @Override + public void update(long value) { + counter.increment(); + histogram.add(value, 1); + } + + public long getCount() { + return counter.getCount(); + } + + public long getMax() { + return this.histogram.getMax(); + } + + public Snapshot snapshot() { + return histogram.snapshotAndReset(); + } +} diff --git hbase-metrics/src/main/java/org/apache/hadoop/hbase/metrics/impl/MetricRegistriesImpl.java hbase-metrics/src/main/java/org/apache/hadoop/hbase/metrics/impl/MetricRegistriesImpl.java new file mode 100644 index 0000000..543ed1e --- /dev/null +++ hbase-metrics/src/main/java/org/apache/hadoop/hbase/metrics/impl/MetricRegistriesImpl.java @@ -0,0 +1,75 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.apache.hadoop.hbase.metrics.impl; + +import java.util.Collection; +import java.util.Collections; +import java.util.Optional; +import java.util.Set; + +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.metrics.MetricRegistries; +import org.apache.hadoop.hbase.metrics.MetricRegistry; +import org.apache.hadoop.hbase.metrics.MetricRegistryFactory; +import org.apache.hadoop.hbase.metrics.MetricRegistryInfo; + +/** + * Implementation of MetricRegistries that does ref-counting. + */ +@InterfaceAudience.Private +public class MetricRegistriesImpl extends MetricRegistries { + private final MetricRegistryFactory factory; + private final RefCountingMap registries; + + public MetricRegistriesImpl() { + this(new MetricRegistryFactoryImpl()); + } + + public MetricRegistriesImpl(MetricRegistryFactory factory) { + this.factory = factory; + this.registries = new RefCountingMap<>(); + } + + @Override + public MetricRegistry create(MetricRegistryInfo info) { + return registries.put(info, () -> factory.create(info)); + } + + public boolean remove(MetricRegistryInfo key) { + return registries.remove(key) == null; + } + + public Optional get(MetricRegistryInfo info) { + return Optional.ofNullable(registries.get(info)); + } + + public Collection getMetricRegistries() { + return Collections.unmodifiableCollection(registries.values()); + } + + public void clear() { + registries.clear(); + } + + public Set getMetricRegistryInfos() { + return Collections.unmodifiableSet(registries.keySet()); + } +} diff --git hbase-metrics/src/main/java/org/apache/hadoop/hbase/metrics/impl/MetricRegistryFactoryImpl.java hbase-metrics/src/main/java/org/apache/hadoop/hbase/metrics/impl/MetricRegistryFactoryImpl.java new file mode 100644 index 0000000..b47153e --- /dev/null +++ hbase-metrics/src/main/java/org/apache/hadoop/hbase/metrics/impl/MetricRegistryFactoryImpl.java @@ -0,0 +1,34 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.apache.hadoop.hbase.metrics.impl; + +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.metrics.MetricRegistry; +import org.apache.hadoop.hbase.metrics.MetricRegistryFactory; +import org.apache.hadoop.hbase.metrics.MetricRegistryInfo; + +@InterfaceAudience.Private +public class MetricRegistryFactoryImpl implements MetricRegistryFactory { + @Override + public MetricRegistry create(MetricRegistryInfo info) { + return new MetricRegistryImpl(info); + } +} diff --git hbase-metrics/src/main/java/org/apache/hadoop/hbase/metrics/impl/MetricRegistryImpl.java hbase-metrics/src/main/java/org/apache/hadoop/hbase/metrics/impl/MetricRegistryImpl.java new file mode 100644 index 0000000..e7ba755 --- /dev/null +++ hbase-metrics/src/main/java/org/apache/hadoop/hbase/metrics/impl/MetricRegistryImpl.java @@ -0,0 +1,122 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.metrics.impl; + +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.metrics.Counter; +import org.apache.hadoop.hbase.metrics.Gauge; +import org.apache.hadoop.hbase.metrics.Histogram; +import org.apache.hadoop.hbase.metrics.Meter; +import org.apache.hadoop.hbase.metrics.Metric; +import org.apache.hadoop.hbase.metrics.MetricRegistry; +import org.apache.hadoop.hbase.metrics.MetricRegistryInfo; +import org.apache.hadoop.hbase.metrics.MetricSet; +import org.apache.hadoop.hbase.metrics.Timer; +import org.apache.hadoop.hbase.util.CollectionUtils; + +/** + * Custom implementation of {@link MetricRegistry}. + */ +@InterfaceAudience.Private +public class MetricRegistryImpl implements MetricRegistry { + + private final MetricRegistryInfo info; + + private final ConcurrentMap metrics; + + public MetricRegistryImpl(MetricRegistryInfo info) { + this.info = info; + this.metrics = new ConcurrentHashMap<>(); + } + + @Override + public Timer timer(String name) { + return (Timer) CollectionUtils.computeIfAbsent(metrics, name, this::createTimer); + } + + protected Timer createTimer() { + return new TimerImpl(); + } + + @Override + public Histogram histogram(String name) { + return (Histogram) CollectionUtils.computeIfAbsent(metrics, name, this::createHistogram); + } + + protected Histogram createHistogram() { + return new HistogramImpl(); + } + + @Override + public Meter meter(String name) { + return (Meter) CollectionUtils.computeIfAbsent(metrics, name, this::createMeter); + } + + protected Meter createMeter() { + return new DropwizardMeter(); + } + + @Override + public Counter counter(String name) { + return (Counter) CollectionUtils.computeIfAbsent(metrics, name, this::createCounter); + } + + protected Counter createCounter() { + return new CounterImpl(); + } + + @Override + public Optional get(String name) { + return Optional.ofNullable(metrics.get(name)); + } + + @Override + public Metric register(String name, Metric metric) { + return CollectionUtils.computeIfAbsent(metrics, name, () -> metric); + } + + @Override + public Gauge register(String name, Gauge gauge) { + return (Gauge) register(name, (Metric)gauge); + } + + @Override + public void registerAll(MetricSet metricSet) { + metricSet.getMetrics().forEach(this::register); + } + + @Override + public Map getMetrics() { + return metrics; + } + + @Override + public boolean remove(String name) { + return metrics.remove(name) != null; + } + + @Override + public MetricRegistryInfo getMetricRegistryInfo() { + return info; + } +} \ No newline at end of file diff --git hbase-metrics/src/main/java/org/apache/hadoop/hbase/metrics/impl/RefCountingMap.java hbase-metrics/src/main/java/org/apache/hadoop/hbase/metrics/impl/RefCountingMap.java new file mode 100644 index 0000000..b185175 --- /dev/null +++ hbase-metrics/src/main/java/org/apache/hadoop/hbase/metrics/impl/RefCountingMap.java @@ -0,0 +1,91 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.apache.hadoop.hbase.metrics.impl; + +import java.util.Collection; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Supplier; +import java.util.stream.Collectors; + +import org.apache.hadoop.hbase.classification.InterfaceAudience; + +/** + * A map of K to V, but does ref counting for added and removed values. The values are + * not added directly, but instead requested from the given Supplier if ref count == 0. Each put() + * call will increment the ref count, and each remove() will decrement it. The values are removed + * from the map iff ref count == 0. + */ +@InterfaceAudience.Private +class RefCountingMap { + + private ConcurrentHashMap> map = new ConcurrentHashMap<>(); + private static class Payload { + V v; + int refCount; + Payload(V v) { + this.v = v; + this.refCount = 1; // create with ref count = 1 + } + } + + V put(K k, Supplier supplier) { + return ((Payload)map.compute(k, (k1, oldValue) -> { + if (oldValue != null) { + oldValue.refCount++; + return oldValue; + } else { + return new Payload(supplier.get()); + } + })).v; + } + + V get(K k) { + Payload p = map.get(k); + return p == null ? null : p.v; + } + + /** + * Decrements the ref count of k, and removes from map if ref count == 0. + * @param k the key to remove + * @return the value associated with the specified key or null if key is removed from map. + */ + V remove(K k) { + Payload p = map.computeIfPresent(k, (k1, v) -> --v.refCount <= 0 ? null : v ); + return p == null ? null : p.v; + } + + void clear() { + map.clear(); + } + + Set keySet() { + return map.keySet(); + } + + Collection values() { + return map.values().stream().map(v -> v.v).collect(Collectors.toList()); + } + + int size() { + return map.size(); + } +} diff --git hbase-metrics/src/main/java/org/apache/hadoop/hbase/metrics/impl/TimerImpl.java hbase-metrics/src/main/java/org/apache/hadoop/hbase/metrics/impl/TimerImpl.java new file mode 100644 index 0000000..df2cc84 --- /dev/null +++ hbase-metrics/src/main/java/org/apache/hadoop/hbase/metrics/impl/TimerImpl.java @@ -0,0 +1,58 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.metrics.impl; + +import java.util.concurrent.TimeUnit; + +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.metrics.Timer; + +/** + * Custom implementation of {@link Timer}. + */ +@InterfaceAudience.Private +public class TimerImpl implements Timer { + private final HistogramImpl histogram; + private final DropwizardMeter meter; + + // track time events in micros + private static final TimeUnit DEFAULT_UNIT = TimeUnit.MICROSECONDS; + + public TimerImpl() { + this.histogram = new HistogramImpl(); + this.meter = new DropwizardMeter(); + } + + @Override + public void update(long duration, TimeUnit unit) { + if (duration >= 0) { + histogram.update(DEFAULT_UNIT.convert(duration, unit)); + meter.mark(); + } + } + + @Override + public HistogramImpl getHistogram() { + return histogram; + } + + @Override + public DropwizardMeter getMeter() { + return meter; + } +} \ No newline at end of file diff --git hbase-metrics/src/main/java/org/apache/hadoop/hbase/metrics/impl/package-info.java hbase-metrics/src/main/java/org/apache/hadoop/hbase/metrics/impl/package-info.java new file mode 100644 index 0000000..9c2e952 --- /dev/null +++ hbase-metrics/src/main/java/org/apache/hadoop/hbase/metrics/impl/package-info.java @@ -0,0 +1,25 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Implementation of the HBase Metrics framework. + */ +@PackageMarker +package org.apache.hadoop.hbase.metrics.impl; + +import org.apache.hadoop.hbase.metrics.PackageMarker; \ No newline at end of file diff --git hbase-metrics/src/main/resources/META-INF/services/org.apache.hadoop.hbase.metrics.MetricRegistries hbase-metrics/src/main/resources/META-INF/services/org.apache.hadoop.hbase.metrics.MetricRegistries new file mode 100644 index 0000000..02edf2e --- /dev/null +++ hbase-metrics/src/main/resources/META-INF/services/org.apache.hadoop.hbase.metrics.MetricRegistries @@ -0,0 +1,18 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl diff --git hbase-metrics/src/test/java/org/apache/hadoop/hbase/metrics/impl/TestCounterImpl.java hbase-metrics/src/test/java/org/apache/hadoop/hbase/metrics/impl/TestCounterImpl.java new file mode 100644 index 0000000..f0b4f8c --- /dev/null +++ hbase-metrics/src/test/java/org/apache/hadoop/hbase/metrics/impl/TestCounterImpl.java @@ -0,0 +1,59 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.metrics.impl; + +import static org.junit.Assert.assertEquals; + +import org.apache.hadoop.hbase.metrics.Counter; +import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * Test class for {@link CounterImpl}. + */ +@Category(SmallTests.class) +public class TestCounterImpl { + + private Counter counter; + + @Before public void setup() { + this.counter = new CounterImpl(); + } + + @Test public void testCounting() { + counter.increment(); + assertEquals(1L, counter.getCount()); + counter.increment(); + assertEquals(2L, counter.getCount()); + counter.increment(2L); + assertEquals(4L, counter.getCount()); + counter.increment(-1L); + assertEquals(3L, counter.getCount()); + + counter.decrement(); + assertEquals(2L, counter.getCount()); + counter.decrement(); + assertEquals(1L, counter.getCount()); + counter.decrement(4L); + assertEquals(-3L, counter.getCount()); + counter.decrement(-3L); + assertEquals(0L, counter.getCount()); + } +} \ No newline at end of file diff --git hbase-metrics/src/test/java/org/apache/hadoop/hbase/metrics/impl/TestDropwizardMeter.java hbase-metrics/src/test/java/org/apache/hadoop/hbase/metrics/impl/TestDropwizardMeter.java new file mode 100644 index 0000000..3ba9821 --- /dev/null +++ hbase-metrics/src/test/java/org/apache/hadoop/hbase/metrics/impl/TestDropwizardMeter.java @@ -0,0 +1,51 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.metrics.impl; + +import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.mockito.Mockito; + +import com.codahale.metrics.Meter; + +/** + * Test class for {@link DropwizardMeter}. + */ +@Category(SmallTests.class) +public class TestDropwizardMeter { + + private Meter meter; + + @Before public void setup() { + this.meter = Mockito.mock(Meter.class); + } + + @Test public void test() { + DropwizardMeter dwMeter = new DropwizardMeter(this.meter); + + dwMeter.mark(); + dwMeter.mark(10L); + dwMeter.mark(); + dwMeter.mark(); + + Mockito.verify(meter, Mockito.times(3)).mark(); + Mockito.verify(meter).mark(10L); + } +} \ No newline at end of file diff --git hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestFastLongHistogram.java hbase-metrics/src/test/java/org/apache/hadoop/hbase/metrics/impl/TestFastLongHistogram.java similarity index 95% rename from hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestFastLongHistogram.java rename to hbase-metrics/src/test/java/org/apache/hadoop/hbase/metrics/impl/TestFastLongHistogram.java index d56d143..6024d25 100644 --- hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestFastLongHistogram.java +++ hbase-metrics/src/test/java/org/apache/hadoop/hbase/metrics/impl/TestFastLongHistogram.java @@ -15,7 +15,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.hadoop.hbase.util; +package org.apache.hadoop.hbase.metrics.impl; + +import static org.junit.Assert.assertEquals; import java.util.Arrays; import java.util.Random; @@ -26,8 +28,6 @@ import org.junit.Assert; import org.junit.Test; import org.junit.experimental.categories.Category; -import static org.junit.Assert.assertEquals; - /** * Testcases for FastLongHistogram. */ @@ -51,7 +51,7 @@ public class TestFastLongHistogram { Assert.assertTrue(j + "-th element org: " + VALUES[j] + ", act: " + vals[j], Math.abs(vals[j] - VALUES[j]) <= 10); } - hist.reset(); + hist.snapshotAndReset(); } } @@ -86,7 +86,7 @@ public class TestFastLongHistogram { Assert.assertTrue("Out of possible value", vals[2] >= 900 && vals[2] <= 1100); } - hist.reset(); + hist.snapshotAndReset(); } } @@ -104,7 +104,7 @@ public class TestFastLongHistogram { h.add(Integer.MAX_VALUE, 1); - h.reset(); + h.snapshotAndReset(); for (long v : VALUES) { for (int i = 0; i < 100; i++) { @@ -126,7 +126,7 @@ public class TestFastLongHistogram { hist.add(50, 100); - hist.reset(); + hist.snapshotAndReset(); doTestUniform(hist); } } diff --git hbase-metrics/src/test/java/org/apache/hadoop/hbase/metrics/impl/TestGauge.java hbase-metrics/src/test/java/org/apache/hadoop/hbase/metrics/impl/TestGauge.java new file mode 100644 index 0000000..7927110 --- /dev/null +++ hbase-metrics/src/test/java/org/apache/hadoop/hbase/metrics/impl/TestGauge.java @@ -0,0 +1,61 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.metrics.impl; + +import static org.junit.Assert.assertEquals; + +import java.util.concurrent.atomic.AtomicLong; + +import org.apache.hadoop.hbase.metrics.Gauge; +import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * Test class for {@link Gauge}. + */ +@Category(SmallTests.class) +public class TestGauge { + + @Test + public void testGetValue() { + SimpleGauge gauge = new SimpleGauge(); + + assertEquals(0, (long)gauge.getValue()); + + gauge.setValue(1000L); + + assertEquals(1000L, (long)gauge.getValue()); + } + + /** + * Gauge implementation with a setter. + */ + private static class SimpleGauge implements Gauge { + + private final AtomicLong value = new AtomicLong(0L); + + @Override public Long getValue() { + return this.value.get(); + } + + public void setValue(long value) { + this.value.set(value); + } + } +} \ No newline at end of file diff --git hbase-metrics/src/test/java/org/apache/hadoop/hbase/metrics/impl/TestHistogramImpl.java hbase-metrics/src/test/java/org/apache/hadoop/hbase/metrics/impl/TestHistogramImpl.java new file mode 100644 index 0000000..1ae3c2c --- /dev/null +++ hbase-metrics/src/test/java/org/apache/hadoop/hbase/metrics/impl/TestHistogramImpl.java @@ -0,0 +1,100 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.apache.hadoop.hbase.metrics.impl; + +import static org.junit.Assert.assertEquals; + +import java.util.stream.IntStream; + +import org.apache.hadoop.hbase.metrics.Snapshot; +import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * Test case for {@link HistogramImpl} + */ +@Category(SmallTests.class) +public class TestHistogramImpl { + + @Test + public void testUpdate() { + HistogramImpl histogram = new HistogramImpl(); + + assertEquals(0, histogram.getCount()); + + histogram.update(0); + assertEquals(1, histogram.getCount()); + + histogram.update(10); + assertEquals(2, histogram.getCount()); + + histogram.update(20); + histogram.update(30); + + assertEquals(4, histogram.getCount()); + } + + @Test + public void testSnapshot() { + HistogramImpl histogram = new HistogramImpl(); + IntStream.range(0, 100).forEach(histogram::update); + + Snapshot snapshot = histogram.snapshot(); + + assertEquals(100, snapshot.getCount()); + assertEquals(50, snapshot.getMedian()); + assertEquals(49, snapshot.getMean()); + assertEquals(0, snapshot.getMin()); + assertEquals(99, snapshot.getMax()); + assertEquals(25, snapshot.get25thPercentile()); + assertEquals(75, snapshot.get75thPercentile()); + assertEquals(90, snapshot.get90thPercentile()); + assertEquals(95, snapshot.get95thPercentile()); + assertEquals(98, snapshot.get98thPercentile()); + assertEquals(99, snapshot.get99thPercentile()); + assertEquals(99, snapshot.get999thPercentile()); + + assertEquals(51, snapshot.getCountAtOrBelow(50)); + + // check that histogram is reset. + assertEquals(100, histogram.getCount()); // count does not reset + + // put more data after reset + IntStream.range(100, 200).forEach(histogram::update); + + assertEquals(200, histogram.getCount()); + + snapshot = histogram.snapshot(); + assertEquals(100, snapshot.getCount()); // only 100 more events + assertEquals(150, snapshot.getMedian()); + assertEquals(149, snapshot.getMean()); + assertEquals(100, snapshot.getMin()); + assertEquals(199, snapshot.getMax()); + assertEquals(125, snapshot.get25thPercentile()); + assertEquals(175, snapshot.get75thPercentile()); + assertEquals(190, snapshot.get90thPercentile()); + assertEquals(195, snapshot.get95thPercentile()); + assertEquals(198, snapshot.get98thPercentile()); + assertEquals(199, snapshot.get99thPercentile()); + assertEquals(199, snapshot.get999thPercentile()); + } +} diff --git hbase-metrics/src/test/java/org/apache/hadoop/hbase/metrics/impl/TestMetricRegistryImpl.java hbase-metrics/src/test/java/org/apache/hadoop/hbase/metrics/impl/TestMetricRegistryImpl.java new file mode 100644 index 0000000..bf08617 --- /dev/null +++ hbase-metrics/src/test/java/org/apache/hadoop/hbase/metrics/impl/TestMetricRegistryImpl.java @@ -0,0 +1,142 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.apache.hadoop.hbase.metrics.impl; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +import java.util.Map; +import java.util.Optional; + +import org.apache.hadoop.hbase.metrics.Counter; +import org.apache.hadoop.hbase.metrics.Gauge; +import org.apache.hadoop.hbase.metrics.Meter; +import org.apache.hadoop.hbase.metrics.Metric; +import org.apache.hadoop.hbase.metrics.MetricRegistryInfo; +import org.apache.hadoop.hbase.metrics.Timer; +import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(SmallTests.class) +public class TestMetricRegistryImpl { + private MetricRegistryInfo info; + private MetricRegistryImpl registry; + + @Before + public void setUp() { + info = new MetricRegistryInfo("foo", "bar", "baz", "foobar", false); + registry = new MetricRegistryImpl(info); + } + + @Test + public void testCounter() { + Counter counter = registry.counter("mycounter"); + assertNotNull(counter); + counter.increment(42L); + Optional metric = registry.get("mycounter"); + assertTrue(metric.isPresent()); + assertEquals(42L, (long)((Counter)metric.get()).getCount()); + } + + @Test + public void testRegisterGauge() { + registry.register("mygauge", new Gauge() { + @Override + public Long getValue() { + return 42L; + } + }); + Optional metric = registry.get("mygauge"); + assertTrue(metric.isPresent()); + assertEquals(42L, (long)((Gauge)metric.get()).getValue()); + } + + @Test + public void testRegisterGaugeLambda() { + // register a Gauge using lambda expression + registry.register("gaugeLambda", () -> 42L); + Optional metric = registry.get("gaugeLambda"); + assertTrue(metric.isPresent()); + assertEquals(42L, (long)((Gauge)metric.get()).getValue()); + } + + @Test + public void testTimer() { + Timer timer = registry.timer("mytimer"); + assertNotNull(timer); + timer.updateNanos(100); + } + + @Test + public void testMeter() { + Meter meter = registry.meter("mymeter"); + assertNotNull(meter); + meter.mark(); + } + + @Test + public void testRegister() { + CounterImpl counter = new CounterImpl(); + registry.register("mycounter", counter); + counter.increment(42L); + + Optional metric = registry.get("mycounter"); + assertTrue(metric.isPresent()); + assertEquals(42L, (long)((Counter)metric.get()).getCount()); + } + + @Test + public void testDoubleRegister() { + Gauge g1 = registry.register("mygauge", () -> 42L); + Gauge g2 = registry.register("mygauge", () -> 52L); + + // second gauge is ignored if it exists + assertEquals(g1, g2); + + Optional metric = registry.get("mygauge"); + assertTrue(metric.isPresent()); + assertEquals(42L, (long)((Gauge)metric.get()).getValue()); + + + Counter c1 = registry.counter("mycounter"); + Counter c2 = registry.counter("mycounter"); + + assertEquals(c1, c2); + } + + @Test + public void testGetMetrics() { + CounterImpl counter = new CounterImpl(); + registry.register("mycounter", counter); + Gauge gauge = registry.register("mygauge", () -> 42L); + Timer timer = registry.timer("mytimer"); + + Map metrics = registry.getMetrics(); + assertEquals(3, metrics.size()); + + assertEquals(counter, metrics.get("mycounter")); + assertEquals(gauge, metrics.get("mygauge")); + assertEquals(timer, metrics.get("mytimer")); + } +} diff --git hbase-metrics/src/test/java/org/apache/hadoop/hbase/metrics/impl/TestRefCountingMap.java hbase-metrics/src/test/java/org/apache/hadoop/hbase/metrics/impl/TestRefCountingMap.java new file mode 100644 index 0000000..cf31d50 --- /dev/null +++ hbase-metrics/src/test/java/org/apache/hadoop/hbase/metrics/impl/TestRefCountingMap.java @@ -0,0 +1,154 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.apache.hadoop.hbase.metrics.impl; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +import java.util.Collection; +import java.util.Set; + +import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import com.google.common.collect.Lists; + +@Category(SmallTests.class) +public class TestRefCountingMap { + + private RefCountingMap map; + + @Before + public void setUp() { + map = new RefCountingMap<>(); + } + + @Test + public void testPutGet() { + map.put("foo", () -> "foovalue"); + + String v = map.get("foo"); + assertNotNull(v); + assertEquals("foovalue", v); + } + + @Test + public void testPutMulti() { + String v1 = map.put("foo", () -> "foovalue"); + String v2 = map.put("foo", () -> "foovalue2"); + String v3 = map.put("foo", () -> "foovalue3"); + + String v = map.get("foo"); + assertEquals("foovalue", v); + assertEquals(v, v1); + assertEquals(v, v2); + assertEquals(v, v3); + } + + @Test + public void testPutRemove() { + map.put("foo", () -> "foovalue"); + String v = map.remove("foo"); + assertNull(v); + v = map.get("foo"); + assertNull(v); + } + + @Test + public void testPutRemoveMulti() { + map.put("foo", () -> "foovalue"); + map.put("foo", () -> "foovalue2"); + map.put("foo", () -> "foovalue3"); + + // remove 1 + String v = map.remove("foo"); + assertEquals("foovalue", v); + + // remove 2 + v = map.remove("foo"); + assertEquals("foovalue", v); + + // remove 3 + v = map.remove("foo"); + assertNull(v); + v = map.get("foo"); + assertNull(v); + } + + @Test + public void testSize() { + assertEquals(0, map.size()); + + // put a key + map.put("foo", () -> "foovalue"); + assertEquals(1, map.size()); + + // put a different key + map.put("bar", () -> "foovalue2"); + assertEquals(2, map.size()); + + // put the same key again + map.put("bar", () -> "foovalue3"); + assertEquals(2, map.size()); // map should be same size + } + + @Test + public void testClear() { + map.put("foo", () -> "foovalue"); + map.put("bar", () -> "foovalue2"); + map.put("baz", () -> "foovalue3"); + + map.clear(); + + assertEquals(0, map.size()); + } + + + @Test + public void testKeySet() { + map.put("foo", () -> "foovalue"); + map.put("bar", () -> "foovalue2"); + map.put("baz", () -> "foovalue3"); + + Set keys = map.keySet(); + assertEquals(3, keys.size()); + + Lists.newArrayList("foo", "bar", "baz").stream().forEach( v -> assertTrue(keys.contains(v))); + } + + @Test + public void testValues() { + map.put("foo", () -> "foovalue"); + map.put("foo", () -> "foovalue2"); + map.put("bar", () -> "foovalue3"); + map.put("baz", () -> "foovalue4"); + + Collection values = map.values(); + assertEquals(3, values.size()); + + Lists.newArrayList("foovalue", "foovalue3", "foovalue4").stream().forEach( + v -> assertTrue(values.contains(v))); + } +} diff --git hbase-metrics/src/test/java/org/apache/hadoop/hbase/metrics/impl/TestTimerImpl.java hbase-metrics/src/test/java/org/apache/hadoop/hbase/metrics/impl/TestTimerImpl.java new file mode 100644 index 0000000..2b3dc8f --- /dev/null +++ hbase-metrics/src/test/java/org/apache/hadoop/hbase/metrics/impl/TestTimerImpl.java @@ -0,0 +1,53 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.metrics.impl; + +import static org.junit.Assert.assertEquals; + +import java.util.concurrent.TimeUnit; + +import org.apache.hadoop.hbase.metrics.Timer; +import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * Test class for {@link TimerImpl} + */ +@Category(SmallTests.class) +public class TestTimerImpl { + + private Timer timer; + + @Before + public void setup() { + this.timer = new TimerImpl(); + } + + @Test + public void testUpdate() { + timer.update(40, TimeUnit.MILLISECONDS); + timer.update(41, TimeUnit.MILLISECONDS); + timer.update(42, TimeUnit.MILLISECONDS); + assertEquals(3, timer.getHistogram().getCount()); + assertEquals(3, timer.getMeter().getCount()); + + assertEquals(TimeUnit.MILLISECONDS.toMicros(41), timer.getHistogram().snapshot().getMedian()); + } +} \ No newline at end of file diff --git hbase-server/pom.xml hbase-server/pom.xml index 0bdee40..55fc256 100644 --- hbase-server/pom.xml +++ hbase-server/pom.xml @@ -387,6 +387,14 @@ test-jar test + + org.apache.hbase + hbase-metrics-api + + + org.apache.hbase + hbase-metrics + org.apache.hbase diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/MasterCoprocessorEnvironment.java hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/MasterCoprocessorEnvironment.java index abe934a..a6e54e0 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/MasterCoprocessorEnvironment.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/MasterCoprocessorEnvironment.java @@ -24,10 +24,20 @@ import org.apache.hadoop.hbase.classification.InterfaceStability; import org.apache.hadoop.hbase.CoprocessorEnvironment; import org.apache.hadoop.hbase.HBaseInterfaceAudience; import org.apache.hadoop.hbase.master.MasterServices; +import org.apache.hadoop.hbase.metrics.MetricRegistry; @InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.COPROC) @InterfaceStability.Evolving public interface MasterCoprocessorEnvironment extends CoprocessorEnvironment { /** @return reference to the HMaster services */ MasterServices getMasterServices(); + + /** + * Returns a MetricRegistry that can be used to track metrics at the master level. + * + *

See ExampleMasterObserverWithMetrics class in the hbase-examples modules for examples + * of how metrics can be instantiated and used.

+ * @return A MetricRegistry for the coprocessor class to track and export metrics. + */ + MetricRegistry getMetricRegistryForMaster(); } diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/MetricsCoprocessor.java hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/MetricsCoprocessor.java new file mode 100644 index 0000000..d564002 --- /dev/null +++ hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/MetricsCoprocessor.java @@ -0,0 +1,136 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.apache.hadoop.hbase.coprocessor; + +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.metrics.MetricRegistries; +import org.apache.hadoop.hbase.metrics.MetricRegistry; +import org.apache.hadoop.hbase.metrics.MetricRegistryInfo; + +import com.google.common.annotations.VisibleForTesting; + +/** + * Utility class for tracking metrics for various types of coprocessors. Each coprocessor instance + * creates its own MetricRegistry which is exported as an individual MetricSource. MetricRegistries + * are ref counted using the hbase-metric module interfaces. + */ +@InterfaceAudience.Private +public class MetricsCoprocessor { + + // Master coprocessor metrics + private static final String MASTER_COPROC_METRICS_NAME = "Coprocessor.Master"; + private static final String MASTER_COPROC_METRICS_CONTEXT = "master"; + private static final String MASTER_COPROC_METRICS_DESCRIPTION + = "Metrics about HBase MasterObservers"; + private static final String MASTER_COPROC_METRICS_JMX_CONTEXT + = "Master,sub=" + MASTER_COPROC_METRICS_NAME; + + // RegionServer coprocessor metrics + private static final String RS_COPROC_METRICS_NAME = "Coprocessor.RegionServer"; + private static final String RS_COPROC_METRICS_CONTEXT = "regionserver"; + private static final String RS_COPROC_METRICS_DESCRIPTION + = "Metrics about HBase RegionServerObservers"; + private static final String RS_COPROC_METRICS_JMX_CONTEXT = "RegionServer,sub=" + + RS_COPROC_METRICS_NAME; + + // Region coprocessor metrics + private static final String REGION_COPROC_METRICS_NAME = "Coprocessor.Region"; + private static final String REGION_COPROC_METRICS_CONTEXT = "regionserver"; + private static final String REGION_COPROC_METRICS_DESCRIPTION + = "Metrics about HBase RegionObservers"; + private static final String REGION_COPROC_METRICS_JMX_CONTEXT + = "RegionServer,sub=" + REGION_COPROC_METRICS_NAME; + + // WAL coprocessor metrics + private static final String WAL_COPROC_METRICS_NAME = "Coprocessor.WAL"; + private static final String WAL_COPROC_METRICS_CONTEXT = "regionserver"; + private static final String WAL_COPROC_METRICS_DESCRIPTION + = "Metrics about HBase WALObservers"; + private static final String WAL_COPROC_METRICS_JMX_CONTEXT + = "RegionServer,sub=" + WAL_COPROC_METRICS_NAME; + + private static String suffix(String metricName, String cpName) { + return new StringBuilder(metricName) + .append(".") + .append("CP_") + .append(cpName) + .toString(); + } + + @VisibleForTesting + static MetricRegistryInfo createRegistryInfoForMasterCoprocessor(String clazz) { + return new MetricRegistryInfo( + suffix(MASTER_COPROC_METRICS_NAME, clazz), + MASTER_COPROC_METRICS_DESCRIPTION, + suffix(MASTER_COPROC_METRICS_JMX_CONTEXT, clazz), + MASTER_COPROC_METRICS_CONTEXT, false); + } + + public static MetricRegistry createRegistryForMasterCoprocessor(String clazz) { + return MetricRegistries.global().create(createRegistryInfoForMasterCoprocessor(clazz)); + } + + @VisibleForTesting + static MetricRegistryInfo createRegistryInfoForRSCoprocessor(String clazz) { + return new MetricRegistryInfo( + suffix(RS_COPROC_METRICS_NAME, clazz), + RS_COPROC_METRICS_DESCRIPTION, + suffix(RS_COPROC_METRICS_JMX_CONTEXT, clazz), + RS_COPROC_METRICS_CONTEXT, false); + } + + public static MetricRegistry createRegistryForRSCoprocessor(String clazz) { + return MetricRegistries.global().create(createRegistryInfoForRSCoprocessor(clazz)); + } + + @VisibleForTesting + public static MetricRegistryInfo createRegistryInfoForRegionCoprocessor(String clazz) { + return new MetricRegistryInfo( + suffix(REGION_COPROC_METRICS_NAME, clazz), + REGION_COPROC_METRICS_DESCRIPTION, + suffix(REGION_COPROC_METRICS_JMX_CONTEXT, clazz), + REGION_COPROC_METRICS_CONTEXT, false); + } + + public static MetricRegistry createRegistryForRegionCoprocessor(String clazz) { + return MetricRegistries.global().create(createRegistryInfoForRegionCoprocessor(clazz)); + } + + @VisibleForTesting + public static MetricRegistryInfo createRegistryInfoForWALCoprocessor(String clazz) { + return new MetricRegistryInfo( + suffix(WAL_COPROC_METRICS_NAME, clazz), + WAL_COPROC_METRICS_DESCRIPTION, + suffix(WAL_COPROC_METRICS_JMX_CONTEXT, clazz), + WAL_COPROC_METRICS_CONTEXT, false); + } + + public static MetricRegistry createRegistryForWALCoprocessor(String clazz) { + return MetricRegistries.global().create(createRegistryInfoForWALCoprocessor(clazz)); + } + + public static void removeRegistry(MetricRegistry registry) { + if (registry == null) { + return; + } + MetricRegistries.global().remove(registry.getMetricRegistryInfo()); + } +} diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionCoprocessorEnvironment.java hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionCoprocessorEnvironment.java index bdf88af..3566f06 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionCoprocessorEnvironment.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionCoprocessorEnvironment.java @@ -21,11 +21,12 @@ package org.apache.hadoop.hbase.coprocessor; import java.util.concurrent.ConcurrentMap; -import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.hbase.classification.InterfaceStability; import org.apache.hadoop.hbase.CoprocessorEnvironment; import org.apache.hadoop.hbase.HBaseInterfaceAudience; import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.classification.InterfaceStability; +import org.apache.hadoop.hbase.metrics.MetricRegistry; import org.apache.hadoop.hbase.regionserver.Region; import org.apache.hadoop.hbase.regionserver.RegionServerServices; @@ -43,4 +44,23 @@ public interface RegionCoprocessorEnvironment extends CoprocessorEnvironment { /** @return shared data between all instances of this coprocessor */ ConcurrentMap getSharedData(); + + /** + * Returns a MetricRegistry that can be used to track metrics at the region server level. All + * metrics tracked at this level will be shared by all the coprocessor instances + * of the same class in the same region server process. Note that there will be one + * region coprocessor environment per region in the server, but all of these instances will share + * the same MetricRegistry. The metric instances (like Counter, Timer, etc) will also be shared + * among all of the region coprocessor instances. + * + *

See ExampleRegionObserverWithMetrics class in the hbase-examples modules to see examples of how + * metrics can be instantiated and used.

+ * @return A MetricRegistry for the coprocessor class to track and export metrics. + */ + // Note: we are not exposing getMetricRegistryForRegion(). per-region metrics are already costly + // so we do not want to allow coprocessors to export metrics at the region level. We can allow + // getMetricRegistryForTable() to allow coprocessors to track metrics per-table, per-regionserver. + MetricRegistry getMetricRegistryForRegionServer(); + + } diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionServerCoprocessorEnvironment.java hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionServerCoprocessorEnvironment.java index 527df45..f42556a 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionServerCoprocessorEnvironment.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionServerCoprocessorEnvironment.java @@ -19,6 +19,7 @@ package org.apache.hadoop.hbase.coprocessor; import org.apache.hadoop.hbase.CoprocessorEnvironment; +import org.apache.hadoop.hbase.metrics.MetricRegistry; import org.apache.hadoop.hbase.regionserver.RegionServerServices; public interface RegionServerCoprocessorEnvironment extends CoprocessorEnvironment { @@ -28,4 +29,13 @@ public interface RegionServerCoprocessorEnvironment extends CoprocessorEnvironme * @return the region server services */ RegionServerServices getRegionServerServices(); + + /** + * Returns a MetricRegistry that can be used to track metrics at the region server level. + * + *

See ExampleMasterObserverWithMetrics class in the hbase-examples modules for examples + * of how metrics can be instantiated and used.

+ * @return A MetricRegistry for the coprocessor class to track and export metrics. + */ + MetricRegistry getMetricRegistryForRegionServer(); } diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/WALCoprocessorEnvironment.java hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/WALCoprocessorEnvironment.java index a4ce5f1..0865d96 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/WALCoprocessorEnvironment.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/WALCoprocessorEnvironment.java @@ -23,6 +23,7 @@ import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; import org.apache.hadoop.hbase.CoprocessorEnvironment; import org.apache.hadoop.hbase.HBaseInterfaceAudience; +import org.apache.hadoop.hbase.metrics.MetricRegistry; import org.apache.hadoop.hbase.wal.WAL; @InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.COPROC) @@ -30,4 +31,13 @@ import org.apache.hadoop.hbase.wal.WAL; public interface WALCoprocessorEnvironment extends CoprocessorEnvironment { /** @return reference to the region server's WAL */ WAL getWAL(); + + /** + * Returns a MetricRegistry that can be used to track metrics at the region server level. + * + *

See ExampleRegionServerObserverWithMetrics class in the hbase-examples modules for examples + * of how metrics can be instantiated and used.

+ * @return A MetricRegistry for the coprocessor class to track and export metrics. + */ + MetricRegistry getMetricRegistryForRegionServer(); } diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/AgeSnapshot.java hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/AgeSnapshot.java index 4c1ad23..dd3bf25 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/AgeSnapshot.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/AgeSnapshot.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.hbase.io.hfile; -import org.apache.hadoop.hbase.util.FastLongHistogram; +import org.apache.hadoop.hbase.metrics.impl.FastLongHistogram; import org.codehaus.jackson.annotate.JsonIgnoreProperties; /** diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCacheUtil.java hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCacheUtil.java index 3c11149..9335ef6 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCacheUtil.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCacheUtil.java @@ -25,7 +25,7 @@ import java.util.concurrent.ConcurrentSkipListSet; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.util.FastLongHistogram; +import org.apache.hadoop.hbase.metrics.impl.FastLongHistogram; import org.codehaus.jackson.JsonGenerationException; import org.codehaus.jackson.annotate.JsonIgnoreProperties; import org.codehaus.jackson.map.JsonMappingException; diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheStats.java hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheStats.java index 7264eda..76e6d2a 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheStats.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheStats.java @@ -23,7 +23,7 @@ import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.LongAdder; import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.hbase.util.FastLongHistogram; +import org.apache.hadoop.hbase.metrics.impl.FastLongHistogram; /** diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java index 3dec2e8..28a8550 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java @@ -19,8 +19,6 @@ package org.apache.hadoop.hbase.master; -import com.google.common.net.HostAndPort; - import java.io.IOException; import java.util.List; import java.util.Set; @@ -45,15 +43,19 @@ import org.apache.hadoop.hbase.coprocessor.CoprocessorHost; import org.apache.hadoop.hbase.coprocessor.CoprocessorService; import org.apache.hadoop.hbase.coprocessor.MasterCoprocessorEnvironment; import org.apache.hadoop.hbase.coprocessor.MasterObserver; +import org.apache.hadoop.hbase.coprocessor.MetricsCoprocessor; import org.apache.hadoop.hbase.coprocessor.ObserverContext; import org.apache.hadoop.hbase.ipc.RpcServer; import org.apache.hadoop.hbase.master.locking.LockProcedure; import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv; +import org.apache.hadoop.hbase.metrics.MetricRegistry; import org.apache.hadoop.hbase.procedure2.ProcedureExecutor; import org.apache.hadoop.hbase.replication.ReplicationPeerConfig; +import org.apache.hadoop.hbase.security.User; import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.SnapshotDescription; import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.Quotas; -import org.apache.hadoop.hbase.security.User; + +import com.google.common.net.HostAndPort; /** * Provides the coprocessor framework and environment for master oriented @@ -72,8 +74,9 @@ public class MasterCoprocessorHost */ static class MasterEnvironment extends CoprocessorHost.Environment implements MasterCoprocessorEnvironment { - private MasterServices masterServices; - final boolean supportGroupCPs; + private final MasterServices masterServices; + private final boolean supportGroupCPs; + private final MetricRegistry metricRegistry; public MasterEnvironment(final Class implClass, final Coprocessor impl, final int priority, final int seq, final Configuration conf, @@ -82,11 +85,25 @@ public class MasterCoprocessorHost this.masterServices = services; supportGroupCPs = !useLegacyMethod(impl.getClass(), "preBalanceRSGroup", ObserverContext.class, String.class); + this.metricRegistry = + MetricsCoprocessor.createRegistryForMasterCoprocessor(implClass.getName()); } + @Override public MasterServices getMasterServices() { return masterServices; } + + @Override + public MetricRegistry getMetricRegistryForMaster() { + return metricRegistry; + } + + @Override + protected void shutdown() { + super.shutdown(); + MetricsCoprocessor.removeRegistry(this.metricRegistry); + } } private MasterServices masterServices; diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServer.java hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServer.java index 8bca6c5..4c28763 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServer.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServer.java @@ -17,10 +17,14 @@ */ package org.apache.hadoop.hbase.regionserver; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.hbase.CompatibilitySingletonFactory; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; -import org.apache.hadoop.hbase.CompatibilitySingletonFactory; +import org.apache.hadoop.hbase.metrics.MetricRegistries; +import org.apache.hadoop.hbase.metrics.MetricRegistry; +import org.apache.hadoop.hbase.metrics.Timer; + +import com.google.common.annotations.VisibleForTesting; /** *

@@ -36,11 +40,20 @@ public class MetricsRegionServer { private MetricsRegionServerSource serverSource; private MetricsRegionServerWrapper regionServerWrapper; + private MetricRegistry metricRegistry; + private Timer bulkLoadTimer; + public MetricsRegionServer(MetricsRegionServerWrapper regionServerWrapper) { this(regionServerWrapper, CompatibilitySingletonFactory.getInstance(MetricsRegionServerSourceFactory.class) .createServer(regionServerWrapper)); + // Create hbase-metrics module based metrics. The registry should already be registered by the + // MetricsRegionServerSource + metricRegistry = MetricRegistries.global().get(serverSource.getMetricRegistryInfo()).get(); + + // create and use metrics from the new hbase-metrics based registry. + bulkLoadTimer = metricRegistry.timer("Bulkload"); } MetricsRegionServer(MetricsRegionServerWrapper regionServerWrapper, @@ -131,4 +144,8 @@ public class MetricsRegionServer { serverSource.updateCompactionInputSize(isMajor, inputBytes); serverSource.updateCompactionOutputSize(isMajor, outputBytes); } + + public void updateBulkLoad(long millis) { + this.bulkLoadTimer.updateMillis(millis); + } } diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java index 1e9f16b..36c4d11 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java @@ -96,6 +96,27 @@ import org.apache.hadoop.hbase.ipc.RpcServerInterface; import org.apache.hadoop.hbase.ipc.ServerNotRunningYetException; import org.apache.hadoop.hbase.ipc.ServerRpcController; import org.apache.hadoop.hbase.master.MasterRpcServices; +import org.apache.hadoop.hbase.quotas.OperationQuota; +import org.apache.hadoop.hbase.quotas.RegionServerQuotaManager; +import org.apache.hadoop.hbase.regionserver.HRegion.RegionScannerImpl; +import org.apache.hadoop.hbase.regionserver.Leases.Lease; +import org.apache.hadoop.hbase.regionserver.Leases.LeaseStillHeldException; +import org.apache.hadoop.hbase.regionserver.Region.Operation; +import org.apache.hadoop.hbase.regionserver.ScannerContext.LimitScope; +import org.apache.hadoop.hbase.regionserver.handler.OpenMetaHandler; +import org.apache.hadoop.hbase.regionserver.handler.OpenPriorityRegionHandler; +import org.apache.hadoop.hbase.regionserver.handler.OpenRegionHandler; +import org.apache.hadoop.hbase.regionserver.wal.WALEdit; +import org.apache.hadoop.hbase.security.User; +import org.apache.hadoop.hbase.shaded.com.google.protobuf.ByteString; +import org.apache.hadoop.hbase.shaded.com.google.protobuf.Message; +import org.apache.hadoop.hbase.shaded.com.google.protobuf.RpcController; +import org.apache.hadoop.hbase.shaded.com.google.protobuf.ServiceException; +import org.apache.hadoop.hbase.shaded.com.google.protobuf.TextFormat; +import org.apache.hadoop.hbase.shaded.com.google.protobuf.UnsafeByteOperations; +import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; +import org.apache.hadoop.hbase.shaded.protobuf.RequestConverter; +import org.apache.hadoop.hbase.shaded.protobuf.ResponseConverter; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.AdminService; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.CloseRegionForSplitOrMergeRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.CloseRegionForSplitOrMergeResponse; @@ -134,9 +155,6 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.UpdateFavor import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.WALEntry; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.WarmupRegionRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.WarmupRegionResponse; -import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; -import org.apache.hadoop.hbase.shaded.protobuf.RequestConverter; -import org.apache.hadoop.hbase.shaded.protobuf.ResponseConverter; import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos; import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.Action; import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.BulkLoadHFileRequest; @@ -177,18 +195,6 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.BulkLoadDescr import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.CompactionDescriptor; import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.FlushDescriptor; import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.RegionEventDescriptor; -import org.apache.hadoop.hbase.quotas.OperationQuota; -import org.apache.hadoop.hbase.quotas.RegionServerQuotaManager; -import org.apache.hadoop.hbase.regionserver.HRegion.RegionScannerImpl; -import org.apache.hadoop.hbase.regionserver.Leases.Lease; -import org.apache.hadoop.hbase.regionserver.Leases.LeaseStillHeldException; -import org.apache.hadoop.hbase.regionserver.Region.Operation; -import org.apache.hadoop.hbase.regionserver.ScannerContext.LimitScope; -import org.apache.hadoop.hbase.regionserver.handler.OpenMetaHandler; -import org.apache.hadoop.hbase.regionserver.handler.OpenPriorityRegionHandler; -import org.apache.hadoop.hbase.regionserver.handler.OpenRegionHandler; -import org.apache.hadoop.hbase.regionserver.wal.WALEdit; -import org.apache.hadoop.hbase.security.User; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.DNS; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; @@ -201,12 +207,7 @@ import org.apache.hadoop.hbase.wal.WALSplitter; import org.apache.hadoop.hbase.zookeeper.ZKSplitLog; import org.apache.zookeeper.KeeperException; -import org.apache.hadoop.hbase.shaded.com.google.protobuf.ByteString; -import org.apache.hadoop.hbase.shaded.com.google.protobuf.Message; -import org.apache.hadoop.hbase.shaded.com.google.protobuf.RpcController; -import org.apache.hadoop.hbase.shaded.com.google.protobuf.ServiceException; -import org.apache.hadoop.hbase.shaded.com.google.protobuf.TextFormat; -import org.apache.hadoop.hbase.shaded.com.google.protobuf.UnsafeByteOperations; +import com.google.common.annotations.VisibleForTesting; /** * Implements the regionserver RPC services. @@ -2104,6 +2105,7 @@ public class RSRpcServices implements HBaseRPCErrorHandler, @Override public BulkLoadHFileResponse bulkLoadHFile(final RpcController controller, final BulkLoadHFileRequest request) throws ServiceException { + long start = EnvironmentEdgeManager.currentTime(); try { checkOpen(); requestCount.increment(); @@ -2148,6 +2150,11 @@ public class RSRpcServices implements HBaseRPCErrorHandler, return builder.build(); } catch (IOException ie) { throw new ServiceException(ie); + } finally { + if (regionServer.metricsRegionServer != null) { + regionServer.metricsRegionServer.updateBulkLoad( + EnvironmentEdgeManager.currentTime() - start); + } } } diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java index 94508b6..e4b47bc 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java @@ -19,11 +19,6 @@ package org.apache.hadoop.hbase.regionserver; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.Lists; -import com.google.protobuf.Message; -import com.google.protobuf.Service; - import java.io.IOException; import java.util.ArrayList; import java.util.List; @@ -44,7 +39,6 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.Coprocessor; -import org.apache.hadoop.hbase.CoprocessorEnvironment; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HBaseInterfaceAudience; import org.apache.hadoop.hbase.HConstants; @@ -65,6 +59,7 @@ import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver; import org.apache.hadoop.hbase.coprocessor.CoprocessorHost; import org.apache.hadoop.hbase.coprocessor.CoprocessorService; import org.apache.hadoop.hbase.coprocessor.EndpointObserver; +import org.apache.hadoop.hbase.coprocessor.MetricsCoprocessor; import org.apache.hadoop.hbase.coprocessor.ObserverContext; import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment; import org.apache.hadoop.hbase.coprocessor.RegionObserver; @@ -75,6 +70,7 @@ import org.apache.hadoop.hbase.io.FSDataInputStreamWrapper; import org.apache.hadoop.hbase.io.Reference; import org.apache.hadoop.hbase.io.hfile.CacheConfig; import org.apache.hadoop.hbase.ipc.RpcServer; +import org.apache.hadoop.hbase.metrics.MetricRegistry; import org.apache.hadoop.hbase.regionserver.Region.Operation; import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest; import org.apache.hadoop.hbase.regionserver.querymatcher.DeleteTracker; @@ -85,6 +81,11 @@ import org.apache.hadoop.hbase.util.CoprocessorClassLoader; import org.apache.hadoop.hbase.util.Pair; import org.apache.hadoop.hbase.wal.WALKey; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Lists; +import com.google.protobuf.Message; +import com.google.protobuf.Service; + /** * Implements the coprocessor environment and runtime support for coprocessors * loaded within a {@link Region}. @@ -112,6 +113,7 @@ public class RegionCoprocessorHost private Region region; private RegionServerServices rsServices; ConcurrentMap sharedData; + private final MetricRegistry metricRegistry; /** * Constructor @@ -125,6 +127,8 @@ public class RegionCoprocessorHost this.region = region; this.rsServices = services; this.sharedData = sharedData; + this.metricRegistry = + MetricsCoprocessor.createRegistryForRegionCoprocessor(impl.getClass().getName()); } /** @return the region */ @@ -141,6 +145,7 @@ public class RegionCoprocessorHost public void shutdown() { super.shutdown(); + MetricsCoprocessor.removeRegistry(this.metricRegistry); } @Override @@ -153,6 +158,10 @@ public class RegionCoprocessorHost return region.getRegionInfo(); } + @Override + public MetricRegistry getMetricRegistryForRegionServer() { + return metricRegistry; + } } static class TableCoprocessorAttribute { diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerCoprocessorHost.java hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerCoprocessorHost.java index f0dd0d7..7732827 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerCoprocessorHost.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerCoprocessorHost.java @@ -25,24 +25,26 @@ import java.util.List; import org.apache.commons.lang.ClassUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.hbase.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.CellScanner; import org.apache.hadoop.hbase.Coprocessor; import org.apache.hadoop.hbase.CoprocessorEnvironment; import org.apache.hadoop.hbase.HBaseInterfaceAudience; import org.apache.hadoop.hbase.MetaMutationAnnotation; +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.classification.InterfaceStability; import org.apache.hadoop.hbase.client.Mutation; import org.apache.hadoop.hbase.coprocessor.CoprocessorHost; +import org.apache.hadoop.hbase.coprocessor.MetricsCoprocessor; import org.apache.hadoop.hbase.coprocessor.ObserverContext; import org.apache.hadoop.hbase.coprocessor.RegionServerCoprocessorEnvironment; import org.apache.hadoop.hbase.coprocessor.RegionServerObserver; import org.apache.hadoop.hbase.coprocessor.SingletonCoprocessorService; import org.apache.hadoop.hbase.ipc.RpcServer; -import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.WALEntry; +import org.apache.hadoop.hbase.metrics.MetricRegistry; import org.apache.hadoop.hbase.replication.ReplicationEndpoint; import org.apache.hadoop.hbase.security.User; +import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.WALEntry; @InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.COPROC) @InterfaceStability.Evolving @@ -335,8 +337,8 @@ public class RegionServerCoprocessorHost extends */ static class RegionServerEnvironment extends CoprocessorHost.Environment implements RegionServerCoprocessorEnvironment { - - private RegionServerServices regionServerServices; + private final RegionServerServices regionServerServices; + private final MetricRegistry metricRegistry; @edu.umd.cs.findbugs.annotations.SuppressWarnings(value="BC_UNCONFIRMED_CAST", justification="Intentional; FB has trouble detecting isAssignableFrom") @@ -353,12 +355,25 @@ public class RegionServerCoprocessorHost extends break; } } + this.metricRegistry = + MetricsCoprocessor.createRegistryForRSCoprocessor(implClass.getName()); } @Override public RegionServerServices getRegionServerServices() { return regionServerServices; } + + @Override + public MetricRegistry getMetricRegistryForRegionServer() { + return metricRegistry; + } + + @Override + protected void shutdown() { + super.shutdown(); + MetricsCoprocessor.removeRegistry(metricRegistry); + } } /** @@ -367,6 +382,7 @@ public class RegionServerCoprocessorHost extends */ static class EnvironmentPriorityComparator implements Comparator { + @Override public int compare(final CoprocessorEnvironment env1, final CoprocessorEnvironment env2) { if (env1.getPriority() < env2.getPriority()) { diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALCoprocessorHost.java hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALCoprocessorHost.java index 4d7e868..a531e83 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALCoprocessorHost.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALCoprocessorHost.java @@ -29,9 +29,11 @@ import org.apache.hadoop.hbase.Coprocessor; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.coprocessor.CoprocessorHost; +import org.apache.hadoop.hbase.coprocessor.MetricsCoprocessor; import org.apache.hadoop.hbase.coprocessor.ObserverContext; import org.apache.hadoop.hbase.coprocessor.WALCoprocessorEnvironment; import org.apache.hadoop.hbase.coprocessor.WALObserver; +import org.apache.hadoop.hbase.metrics.MetricRegistry; import org.apache.hadoop.hbase.wal.WAL; import org.apache.hadoop.hbase.wal.WALKey; @@ -51,6 +53,8 @@ public class WALCoprocessorHost private final WAL wal; + private final MetricRegistry metricRegistry; + @Override public WAL getWAL() { return wal; @@ -70,6 +74,18 @@ public class WALCoprocessorHost final WAL wal) { super(impl, priority, seq, conf); this.wal = wal; + this.metricRegistry = MetricsCoprocessor.createRegistryForWALCoprocessor(implClass.getName()); + } + + @Override + public MetricRegistry getMetricRegistryForRegionServer() { + return metricRegistry; + } + + @Override + protected void shutdown() { + super.shutdown(); + MetricsCoprocessor.removeRegistry(this.metricRegistry); } } diff --git hbase-server/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java hbase-server/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java index 190de4e..a8887d4 100644 --- hbase-server/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java +++ hbase-server/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java @@ -660,6 +660,13 @@ public class MiniHBaseCluster extends HBaseCluster { return hbaseCluster.getRegionServer(serverNumber); } + public HRegionServer getRegionServer(ServerName serverName) { + return hbaseCluster.getRegionServers().stream() + .map(t -> t.getRegionServer()) + .filter(r -> r.getServerName().equals(serverName)) + .findFirst().orElse(null); + } + public List getRegions(byte[] tableName) { return getRegions(TableName.valueOf(tableName)); } diff --git hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorMetrics.java hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorMetrics.java new file mode 100644 index 0000000..50e97ba --- /dev/null +++ hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorMetrics.java @@ -0,0 +1,544 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.coprocessor; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.util.List; +import java.util.Optional; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.CoprocessorEnvironment; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HRegionLocation; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.client.Connection; +import org.apache.hadoop.hbase.client.ConnectionFactory; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.Mutation; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.RegionLocator; +import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.ipc.CoprocessorRpcChannel; +import org.apache.hadoop.hbase.metrics.Counter; +import org.apache.hadoop.hbase.metrics.Metric; +import org.apache.hadoop.hbase.metrics.MetricRegistries; +import org.apache.hadoop.hbase.metrics.MetricRegistry; +import org.apache.hadoop.hbase.metrics.MetricRegistryInfo; +import org.apache.hadoop.hbase.metrics.Timer; +import org.apache.hadoop.hbase.protobuf.ProtobufUtil; +import org.apache.hadoop.hbase.protobuf.generated.ClientProtos; +import org.apache.hadoop.hbase.protobuf.generated.MultiRowMutationProtos.MultiRowMutationService; +import org.apache.hadoop.hbase.protobuf.generated.MultiRowMutationProtos.MutateRowsRequest; +import org.apache.hadoop.hbase.protobuf.generated.MultiRowMutationProtos.MutateRowsResponse; +import org.apache.hadoop.hbase.regionserver.HRegionServer; +import org.apache.hadoop.hbase.regionserver.wal.WALEdit; +import org.apache.hadoop.hbase.testclassification.CoprocessorTests; +import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import com.google.common.collect.Lists; +import com.google.protobuf.RpcCallback; +import com.google.protobuf.RpcController; +import com.google.protobuf.ServiceException; + +/** + * Testing of coprocessor metrics end-to-end. + */ +@Category({CoprocessorTests.class, MediumTests.class}) +public class TestCoprocessorMetrics { + + private static final Log LOG = LogFactory.getLog(TestCoprocessorMetrics.class); + private static final HBaseTestingUtility UTIL = new HBaseTestingUtility(); + + private static final byte[] foo = Bytes.toBytes("foo"); + private static final byte[] bar = Bytes.toBytes("bar"); + /** + * MasterObserver that has a Timer metric for create table operation. + */ + public static class CustomMasterObserver extends BaseMasterObserver { + private Timer createTableTimer; + private long start = Long.MIN_VALUE; + + @Override + public void preCreateTable(ObserverContext ctx, + HTableDescriptor desc, HRegionInfo[] regions) throws IOException { + super.preCreateTable(ctx, desc, regions); + + // we rely on the fact that there is only 1 instance of our MasterObserver + this.start = System.currentTimeMillis(); + } + + @Override + public void postCreateTable(ObserverContext ctx, + HTableDescriptor desc, HRegionInfo[] regions) throws IOException { + super.postCreateTable(ctx, desc, regions); + if (this.start > 0) { + long time = System.currentTimeMillis() - start; + LOG.info("Create table took: " + time); + createTableTimer.updateMillis(time); + } + } + + @Override + public void start(CoprocessorEnvironment env) throws IOException { + super.start(env); + if (env instanceof MasterCoprocessorEnvironment) { + MetricRegistry registry = + ((MasterCoprocessorEnvironment) env).getMetricRegistryForMaster(); + + createTableTimer = registry.timer("CreateTable"); + } + } + } + + /** + * RegionServerObserver that has a Counter for rollWAL requests. + */ + public static class CustomRegionServerObserver extends BaseRegionServerObserver { + /** This is the Counter metric object to keep track of the current count across invocations */ + private Counter rollWALCounter; + @Override + public void postRollWALWriterRequest(ObserverContext ctx) + throws IOException { + // Increment the Counter whenever the coprocessor is called + rollWALCounter.increment(); + super.postRollWALWriterRequest(ctx); + } + + @Override + public void start(CoprocessorEnvironment env) throws IOException { + super.start(env); + if (env instanceof RegionServerCoprocessorEnvironment) { + MetricRegistry registry = + ((RegionServerCoprocessorEnvironment) env).getMetricRegistryForRegionServer(); + + if (rollWALCounter == null) { + rollWALCounter = registry.counter("rollWALRequests"); + } + } + } + } + + /** + * WALObserver that has a Counter for walEdits written. + */ + public static class CustomWALObserver extends BaseWALObserver { + private Counter walEditsCount; + + @Override + public void postWALWrite(ObserverContext ctx, + HRegionInfo info, org.apache.hadoop.hbase.wal.WALKey logKey, + WALEdit logEdit) throws IOException { + super.postWALWrite(ctx, info, logKey, logEdit); + walEditsCount.increment(); + } + + @Override + public void start(CoprocessorEnvironment env) throws IOException { + super.start(env); + if (env instanceof WALCoprocessorEnvironment) { + MetricRegistry registry = + ((WALCoprocessorEnvironment) env).getMetricRegistryForRegionServer(); + + if (walEditsCount == null) { + walEditsCount = registry.counter("walEditsCount"); + } + } + } + } + + /** + * RegionObserver that has a Counter for preGet() + */ + public static class CustomRegionObserver extends BaseRegionObserver { + private Counter preGetCounter; + + @Override + public void preGetOp(ObserverContext e, Get get, + List results) throws IOException { + super.preGetOp(e, get, results); + preGetCounter.increment(); + } + + @Override + public void start(CoprocessorEnvironment env) throws IOException { + super.start(env); + + if (env instanceof RegionCoprocessorEnvironment) { + MetricRegistry registry = + ((RegionCoprocessorEnvironment) env).getMetricRegistryForRegionServer(); + + if (preGetCounter == null) { + preGetCounter = registry.counter("preGetRequests"); + } + } + } + } + + public static class CustomRegionObserver2 extends CustomRegionObserver { + } + + /** + * RegionEndpoint to test metrics from endpoint calls + */ + public static class CustomRegionEndpoint extends MultiRowMutationEndpoint { + + private Timer endpointExecution; + + @Override + public void mutateRows(RpcController controller, MutateRowsRequest request, + RpcCallback done) { + long start = System.nanoTime(); + super.mutateRows(controller, request, done); + endpointExecution.updateNanos(System.nanoTime() - start); + } + + @Override + public void start(CoprocessorEnvironment env) throws IOException { + super.start(env); + + if (env instanceof RegionCoprocessorEnvironment) { + MetricRegistry registry = + ((RegionCoprocessorEnvironment) env).getMetricRegistryForRegionServer(); + + if (endpointExecution == null) { + endpointExecution = registry.timer("EndpointExecution"); + } + } + } + } + + @BeforeClass + public static void setupBeforeClass() throws Exception { + Configuration conf = UTIL.getConfiguration(); + // inject master, regionserver and WAL coprocessors + conf.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, + CustomMasterObserver.class.getName()); + conf.set(CoprocessorHost.REGIONSERVER_COPROCESSOR_CONF_KEY, + CustomRegionServerObserver.class.getName()); + conf.set(CoprocessorHost.WAL_COPROCESSOR_CONF_KEY, + CustomWALObserver.class.getName()); + conf.setBoolean(CoprocessorHost.ABORT_ON_ERROR_KEY, true); + UTIL.startMiniCluster(); + } + + @AfterClass + public static void teardownAfterClass() throws Exception { + UTIL.shutdownMiniCluster(); + } + + @Before + public void setup() throws IOException { + try (Connection connection = ConnectionFactory.createConnection(UTIL.getConfiguration()); + Admin admin = connection.getAdmin()) { + for (HTableDescriptor htd : admin.listTables()) { + UTIL.deleteTable(htd.getTableName()); + } + } + } + + @Test + public void testMasterObserver() throws IOException { + // Find out the MetricRegistry used by the CP using the global registries + MetricRegistryInfo info = MetricsCoprocessor.createRegistryInfoForMasterCoprocessor( + CustomMasterObserver.class.getName()); + Optional registry = MetricRegistries.global().get(info); + assertTrue(registry.isPresent()); + + Optional metric = registry.get().get("CreateTable"); + assertTrue(metric.isPresent()); + + try (Connection connection = ConnectionFactory.createConnection(UTIL.getConfiguration()); + Admin admin = connection.getAdmin()) { + + Timer createTableTimer = (Timer)metric.get(); + long prevCount = createTableTimer.getHistogram().getCount(); + LOG.info("Creating table"); + admin.createTable( + new HTableDescriptor("testMasterObserver") + .addFamily(new HColumnDescriptor("foo"))); + + assertEquals(1, createTableTimer.getHistogram().getCount() - prevCount); + } + } + + @Test + public void testRegionServerObserver() throws IOException { + try (Connection connection = ConnectionFactory.createConnection(UTIL.getConfiguration()); + Admin admin = connection.getAdmin()) { + LOG.info("Rolling WALs"); + admin.rollWALWriter(UTIL.getMiniHBaseCluster().getServerHoldingMeta()); + } + + // Find out the MetricRegistry used by the CP using the global registries + MetricRegistryInfo info = MetricsCoprocessor.createRegistryInfoForRSCoprocessor( + CustomRegionServerObserver.class.getName()); + + Optional registry = MetricRegistries.global().get(info); + assertTrue(registry.isPresent()); + + Optional metric = registry.get().get("rollWALRequests"); + assertTrue(metric.isPresent()); + + Counter rollWalRequests = (Counter)metric.get(); + assertEquals(1, rollWalRequests.getCount()); + } + + @Test + public void testWALObserver() throws IOException { + // Find out the MetricRegistry used by the CP using the global registries + MetricRegistryInfo info = MetricsCoprocessor.createRegistryInfoForWALCoprocessor( + CustomWALObserver.class.getName()); + + Optional registry = MetricRegistries.global().get(info); + assertTrue(registry.isPresent()); + + Optional metric = registry.get().get("walEditsCount"); + assertTrue(metric.isPresent()); + + try (Connection connection = ConnectionFactory.createConnection(UTIL.getConfiguration()); + Admin admin = connection.getAdmin()) { + admin.createTable( + new HTableDescriptor("testWALObserver") + .addFamily(new HColumnDescriptor("foo"))); + + Counter rollWalRequests = (Counter)metric.get(); + long prevCount = rollWalRequests.getCount(); + assertTrue(prevCount > 0); + + try (Table table = connection.getTable(TableName.valueOf("testWALObserver"))) { + table.put(new Put(foo).addColumn(foo, foo, foo)); + } + + assertEquals(1, rollWalRequests.getCount() - prevCount); + } + } + + /** + * Helper for below tests + */ + private void assertPreGetRequestsCounter(Class coprocClass) { + // Find out the MetricRegistry used by the CP using the global registries + MetricRegistryInfo info = MetricsCoprocessor.createRegistryInfoForRegionCoprocessor( + coprocClass.getName()); + + Optional registry = MetricRegistries.global().get(info); + assertTrue(registry.isPresent()); + + Optional metric = registry.get().get("preGetRequests"); + assertTrue(metric.isPresent()); + + Counter preGetRequests = (Counter)metric.get(); + assertEquals(2, preGetRequests.getCount()); + } + + @Test + public void testRegionObserverSingleRegion() throws IOException { + TableName tableName = TableName.valueOf("testRegionObserverSingleRegion"); + try (Connection connection = ConnectionFactory.createConnection(UTIL.getConfiguration()); + Admin admin = connection.getAdmin()) { + admin.createTable( + new HTableDescriptor(tableName) + .addFamily(new HColumnDescriptor(foo)) + // add the coprocessor for the region + .addCoprocessor(CustomRegionObserver.class.getName())); + try (Table table = connection.getTable(tableName)) { + table.get(new Get(foo)); + table.get(new Get(foo)); // 2 gets + } + } + + assertPreGetRequestsCounter(CustomRegionObserver.class); + } + + @Test + public void testRegionObserverMultiRegion() throws IOException { + TableName tableName = TableName.valueOf("testRegionObserverMultiRegion"); + try (Connection connection = ConnectionFactory.createConnection(UTIL.getConfiguration()); + Admin admin = connection.getAdmin()) { + admin.createTable( + new HTableDescriptor(tableName) + .addFamily(new HColumnDescriptor(foo)) + // add the coprocessor for the region + .addCoprocessor(CustomRegionObserver.class.getName()) + , new byte[][]{foo}); // create with 2 regions + try (Table table = connection.getTable(tableName); + RegionLocator locator = connection.getRegionLocator(tableName)) { + table.get(new Get(bar)); + table.get(new Get(foo)); // 2 gets to 2 separate regions + assertEquals(2, locator.getAllRegionLocations().size()); + assertNotEquals(locator.getRegionLocation(bar).getRegionInfo(), + locator.getRegionLocation(foo).getRegionInfo()); + } + } + + assertPreGetRequestsCounter(CustomRegionObserver.class); + } + + @Test + public void testRegionObserverMultiTable() throws IOException { + TableName tableName1 = TableName.valueOf("testRegionObserverMultiTable1"); + TableName tableName2 = TableName.valueOf("testRegionObserverMultiTable2"); + try (Connection connection = ConnectionFactory.createConnection(UTIL.getConfiguration()); + Admin admin = connection.getAdmin()) { + admin.createTable( + new HTableDescriptor(tableName1) + .addFamily(new HColumnDescriptor(foo)) + // add the coprocessor for the region + .addCoprocessor(CustomRegionObserver.class.getName())); + admin.createTable( + new HTableDescriptor(tableName2) + .addFamily(new HColumnDescriptor(foo)) + // add the coprocessor for the region + .addCoprocessor(CustomRegionObserver.class.getName())); + try (Table table1 = connection.getTable(tableName1); + Table table2 = connection.getTable(tableName2);) { + table1.get(new Get(bar)); + table2.get(new Get(foo)); // 2 gets to 2 separate tables + } + } + assertPreGetRequestsCounter(CustomRegionObserver.class); + } + + @Test + public void testRegionObserverMultiCoprocessor() throws IOException { + TableName tableName = TableName.valueOf("testRegionObserverMultiCoprocessor"); + try (Connection connection = ConnectionFactory.createConnection(UTIL.getConfiguration()); + Admin admin = connection.getAdmin()) { + admin.createTable( + new HTableDescriptor(tableName) + .addFamily(new HColumnDescriptor(foo)) + // add the coprocessor for the region. We add two different coprocessors + .addCoprocessor(CustomRegionObserver.class.getName()) + .addCoprocessor(CustomRegionObserver2.class.getName())); + try (Table table = connection.getTable(tableName)) { + table.get(new Get(foo)); + table.get(new Get(foo)); // 2 gets + } + } + + // we will have two counters coming from two coprocs, in two different MetricRegistries + assertPreGetRequestsCounter(CustomRegionObserver.class); + assertPreGetRequestsCounter(CustomRegionObserver2.class); + } + + @Test + public void testRegionObserverAfterRegionClosed() throws IOException { + TableName tableName = TableName.valueOf("testRegionObserverAfterRegionClosed"); + try (Connection connection = ConnectionFactory.createConnection(UTIL.getConfiguration()); + Admin admin = connection.getAdmin()) { + admin.createTable( + new HTableDescriptor(tableName) + .addFamily(new HColumnDescriptor(foo)) + // add the coprocessor for the region + .addCoprocessor(CustomRegionObserver.class.getName()) + , new byte[][]{foo}); // create with 2 regions + try (Table table = connection.getTable(tableName)) { + table.get(new Get(foo)); + table.get(new Get(foo)); // 2 gets + } + + assertPreGetRequestsCounter(CustomRegionObserver.class); + + // close one of the regions + try (RegionLocator locator = connection.getRegionLocator(tableName)) { + HRegionLocation loc = locator.getRegionLocation(foo); + admin.closeRegion(loc.getServerName(), loc.getRegionInfo()); + + HRegionServer server = UTIL.getMiniHBaseCluster().getRegionServer(loc.getServerName()); + UTIL.waitFor(30000, + () -> server.getOnlineRegion(loc.getRegionInfo().getRegionName()) == null); + assertNull(server.getOnlineRegion(loc.getRegionInfo().getRegionName())); + } + + // with only 1 region remaining, we should still be able to find the Counter + assertPreGetRequestsCounter(CustomRegionObserver.class); + + // close the table + admin.disableTable(tableName); + + MetricRegistryInfo info = MetricsCoprocessor.createRegistryInfoForRegionCoprocessor( + CustomRegionObserver.class.getName()); + + // ensure that MetricRegistry is deleted + Optional registry = MetricRegistries.global().get(info); + assertFalse(registry.isPresent()); + } + } + + @Test + public void testRegionObserverEndpoint() throws IOException, ServiceException { + TableName tableName = TableName.valueOf("testRegionObserverEndpoint"); + try (Connection connection = ConnectionFactory.createConnection(UTIL.getConfiguration()); + Admin admin = connection.getAdmin()) { + admin.createTable( + new HTableDescriptor(tableName) + .addFamily(new HColumnDescriptor(foo)) + // add the coprocessor for the region + .addCoprocessor(CustomRegionEndpoint.class.getName())); + + try (Table table = connection.getTable(tableName)) { + List mutations = Lists.newArrayList(new Put(foo), new Put(bar)); + MutateRowsRequest.Builder mrmBuilder = MutateRowsRequest.newBuilder(); + + for (Mutation mutation : mutations) { + mrmBuilder.addMutationRequest(ProtobufUtil.toMutation( + ClientProtos.MutationProto.MutationType.PUT, mutation)); + } + + CoprocessorRpcChannel channel = table.coprocessorService(bar); + MultiRowMutationService.BlockingInterface service = + MultiRowMutationService.newBlockingStub(channel); + MutateRowsRequest mrm = mrmBuilder.build(); + service.mutateRows(null, mrm); + } + } + + // Find out the MetricRegistry used by the CP using the global registries + MetricRegistryInfo info = MetricsCoprocessor.createRegistryInfoForRegionCoprocessor( + CustomRegionEndpoint.class.getName()); + + Optional registry = MetricRegistries.global().get(info); + assertTrue(registry.isPresent()); + + Optional metric = registry.get().get("EndpointExecution"); + assertTrue(metric.isPresent()); + + Timer endpointExecutions = (Timer)metric.get(); + assertEquals(1, endpointExecutions.getHistogram().getCount()); + } +} diff --git hbase-server/src/test/java/org/apache/hadoop/hbase/security/token/TestTokenAuthentication.java hbase-server/src/test/java/org/apache/hadoop/hbase/security/token/TestTokenAuthentication.java index c8fe299..5b46af5 100644 --- hbase-server/src/test/java/org/apache/hadoop/hbase/security/token/TestTokenAuthentication.java +++ hbase-server/src/test/java/org/apache/hadoop/hbase/security/token/TestTokenAuthentication.java @@ -54,6 +54,7 @@ import org.apache.hadoop.hbase.ipc.RpcServer.BlockingServiceAndInterface; import org.apache.hadoop.hbase.ipc.RpcServerFactory; import org.apache.hadoop.hbase.ipc.RpcServerInterface; import org.apache.hadoop.hbase.ipc.ServerRpcController; +import org.apache.hadoop.hbase.metrics.MetricRegistry; import org.apache.hadoop.hbase.protobuf.generated.AuthenticationProtos; import org.apache.hadoop.hbase.regionserver.HRegion; import org.apache.hadoop.hbase.regionserver.RegionServerServices; @@ -260,6 +261,11 @@ public class TestTokenAuthentication { public ConcurrentMap getSharedData() { return null; } @Override + public MetricRegistry getMetricRegistryForRegionServer() { + return null; + } + + @Override public int getVersion() { return 0; } @Override diff --git pom.xml pom.xml index 084629e..637b236 100644 --- pom.xml +++ pom.xml @@ -84,6 +84,8 @@ hbase-shaded hbase-spark hbase-archetypes + hbase-metrics-api + hbase-metrics @@ -1495,6 +1497,30 @@ org.apache.hbase ${project.version} + + hbase-metrics-api + org.apache.hbase + ${project.version} + + + hbase-metrics-api + org.apache.hbase + ${project.version} + test-jar + test + + + hbase-metrics + org.apache.hbase + ${project.version} + + + hbase-metrics + org.apache.hbase + ${project.version} + test-jar + test + com.github.stephenc.findbugs