+ * These metrics will be available through the regular Hadoop metrics2 sinks (ganglia, opentsdb, + * etc) as well as JMX output. You can view a snapshot of the metrics by going to the http web UI + * of the master page, something like http://mymasterhost:16010/jmx + *
+ * @see ExampleRegionObserverWithMetrics + */ +public class ExampleMasterObserverWithMetrics extends BaseMasterObserver { + + private static final Log LOG = LogFactory.getLog(ExampleMasterObserverWithMetrics.class); + + /** This is the Timer metric object to keep track of the current count across invocations */ + private Timer createTableTimer; + private long createTableStartTime = Long.MIN_VALUE; + + /** This is a Counter object to keep track of disableTable operations */ + private Counter disableTableCounter; + + /** Returns the total memory of the process. We will use this to define a gauge metric */ + private long getTotalMemory() { + return Runtime.getRuntime().totalMemory(); + } + + /** Returns the max memory of the process. We will use this to define a gauge metric */ + private long getMaxMemory() { + return Runtime.getRuntime().maxMemory(); + } + + @Override + public void preCreateTable(ObserverContext+ * These metrics will be available through the regular Hadoop metrics2 sinks (ganglia, opentsdb, + * etc) as well as JMX output. You can view a snapshot of the metrics by going to the http web UI + * of the regionserver page, something like http://myregionserverhost:16030/jmx + *
+ * + * @see ExampleMasterObserverWithMetrics + */ +public class ExampleRegionObserverWithMetrics extends BaseRegionObserver { + + private Counter preGetCounter; + private Timer costlyOperationTimer; + + @Override + public void preGetOp(ObserverContextDeveloper note: + * Unlike the current metrics2 based approach, the new metrics approach + * (hbase-metrics-api and hbase-metrics modules) work by having different MetricRegistries that are + * initialized and used from the code that lives in their respective modules (hbase-server, etc). + * There is no need to define BaseSource classes and do a lot of indirection. The MetricRegistry'es + * will be in the global MetricRegistriesImpl, and this class will iterate over + * MetricRegistries.global() and register adapters to the metrics2 subsystem. These adapters then + * report the actual values by delegating to + * {@link HBaseMetrics2HadoopMetricsAdapter#snapshotAllMetrics(MetricRegistry, MetricsCollector)}. + * + * We do not initialize the Hadoop Metrics2 system assuming that other BaseSources already do so + * (see BaseSourceImpl). Once the last BaseSource is moved to the new system, the metric2 + * initialization should be moved here. + *
+ */ +public class GlobalMetricRegistriesAdapter { + + private static final Log LOG = LogFactory.getLog(GlobalMetricRegistriesAdapter.class); + + private class MetricsSourceAdapter implements MetricsSource { + private final MetricRegistry registry; + MetricsSourceAdapter(MetricRegistry registry) { + this.registry = registry; + } + + @Override + public void getMetrics(MetricsCollector collector, boolean all) { + metricsAdapter.snapshotAllMetrics(registry, collector); + } + } + + private final MetricsExecutor executor; + private final AtomicBoolean stopped; + private final DefaultMetricsSystemHelper helper; + private final HBaseMetrics2HadoopMetricsAdapter metricsAdapter; + private final HashMapThe result is maven-compiler-plugin can properly identify the scope of + * changed files + * + *
See more details in
+ *
+ * maven-compiler-plugin: incremental compilation broken
+ */
+@Retention(RetentionPolicy.SOURCE)
+public @interface PackageMarker {
+}
diff --git hbase-metrics-api/src/main/java/org/apache/hadoop/hbase/metrics/Snapshot.java hbase-metrics-api/src/main/java/org/apache/hadoop/hbase/metrics/Snapshot.java
new file mode 100644
index 0000000..56ee8ae
--- /dev/null
+++ hbase-metrics-api/src/main/java/org/apache/hadoop/hbase/metrics/Snapshot.java
@@ -0,0 +1,136 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.hadoop.hbase.metrics;
+
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+
+/**
+ * A statictical sample of histogram values.
+ */
+@InterfaceAudience.Private
+public interface Snapshot {
+
+ /**
+ * Return the values with the given quantiles.
+ * @param quantiles the requested quantiles.
+ * @return the value for the quantiles.
+ */
+ long[] getQuantiles(double[] quantiles);
+
+ /**
+ * Return the values with the default quantiles.
+ * @return the value for default the quantiles.
+ */
+ long[] getQuantiles();
+
+ /**
+ * Returns the number of values in the snapshot.
+ *
+ * @return the number of values
+ */
+ long getCount();
+
+ /**
+ * Returns the total count below the given value
+ * @param val the value
+ * @return the total count below the given value
+ */
+ long getCountAtOrBelow(long val);
+
+ /**
+ * Returns the value at the 25th percentile in the distribution.
+ *
+ * @return the value at the 25th percentile
+ */
+ long get25thPercentile();
+
+ /**
+ * Returns the value at the 75th percentile in the distribution.
+ *
+ * @return the value at the 75th percentile
+ */
+ long get75thPercentile();
+
+ /**
+ * Returns the value at the 90th percentile in the distribution.
+ *
+ * @return the value at the 90th percentile
+ */
+ long get90thPercentile();
+
+ /**
+ * Returns the value at the 95th percentile in the distribution.
+ *
+ * @return the value at the 95th percentile
+ */
+ long get95thPercentile();
+
+ /**
+ * Returns the value at the 98th percentile in the distribution.
+ *
+ * @return the value at the 98th percentile
+ */
+ long get98thPercentile();
+
+ /**
+ * Returns the value at the 99th percentile in the distribution.
+ *
+ * @return the value at the 99th percentile
+ */
+ long get99thPercentile();
+
+ /**
+ * Returns the value at the 99.9th percentile in the distribution.
+ *
+ * @return the value at the 99.9th percentile
+ */
+ long get999thPercentile();
+
+ /**
+ * Returns the median value in the distribution.
+ *
+ * @return the median value
+ */
+ long getMedian();
+
+ /**
+ * Returns the highest value in the snapshot.
+ *
+ * @return the highest value
+ */
+ long getMax();
+
+ /**
+ * Returns the arithmetic mean of the values in the snapshot.
+ *
+ * @return the arithmetic mean
+ */
+ long getMean();
+
+ /**
+ * Returns the lowest value in the snapshot.
+ *
+ * @return the lowest value
+ */
+ long getMin();
+
+ // TODO: Dropwizard histograms also track stddev
+}
diff --git hbase-metrics-api/src/main/java/org/apache/hadoop/hbase/metrics/Timer.java hbase-metrics-api/src/main/java/org/apache/hadoop/hbase/metrics/Timer.java
new file mode 100644
index 0000000..63fc649
--- /dev/null
+++ hbase-metrics-api/src/main/java/org/apache/hadoop/hbase/metrics/Timer.java
@@ -0,0 +1,68 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.metrics;
+
+import java.util.concurrent.TimeUnit;
+
+import org.apache.hadoop.hbase.HBaseInterfaceAudience;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.classification.InterfaceStability;
+
+/**
+ * A metric which encompasses a {@link Histogram} and {@link Meter}.
+ */
+@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.COPROC)
+@InterfaceStability.Evolving
+public interface Timer extends Metric {
+ /**
+ * Update the timer with the given duration in given time unit.
+ * @param duration the duration of the event
+ * @param unit the time unit for the duration
+ */
+ void update(long duration, TimeUnit unit);
+
+ /**
+ * Update the timer with the given duration in milliseconds
+ * @param durationMillis the duration of the event in ms
+ */
+ default void updateMillis(long durationMillis) {
+ update(durationMillis, TimeUnit.NANOSECONDS);
+ }
+
+ /**
+ * Update the timer with the given duration in microseconds
+ * @param durationMicros the duration of the event in microseconds
+ */
+ default void updateMicros(long durationMicros) {
+ update(durationMicros, TimeUnit.MICROSECONDS);
+ }
+
+ /**
+ * Update the timer with the given duration in nanoseconds
+ * @param durationNanos the duration of the event in ns
+ */
+ default void updateNanos(long durationNanos) {
+ update(durationNanos, TimeUnit.NANOSECONDS);
+ }
+
+ @InterfaceAudience.Private
+ Histogram getHistogram();
+
+ @InterfaceAudience.Private
+ Meter getMeter();
+}
\ No newline at end of file
diff --git hbase-metrics-api/src/main/java/org/apache/hadoop/hbase/metrics/package-info.java hbase-metrics-api/src/main/java/org/apache/hadoop/hbase/metrics/package-info.java
new file mode 100644
index 0000000..e79451f
--- /dev/null
+++ hbase-metrics-api/src/main/java/org/apache/hadoop/hbase/metrics/package-info.java
@@ -0,0 +1,25 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Metrics API for HBase.
+ */
+@PackageMarker
+package org.apache.hadoop.hbase.metrics;
+
+// End package-info.java
diff --git hbase-metrics-api/src/test/java/org/apache/hadoop/hbase/metrics/TestMetricRegistriesLoader.java hbase-metrics-api/src/test/java/org/apache/hadoop/hbase/metrics/TestMetricRegistriesLoader.java
new file mode 100644
index 0000000..8746146
--- /dev/null
+++ hbase-metrics-api/src/test/java/org/apache/hadoop/hbase/metrics/TestMetricRegistriesLoader.java
@@ -0,0 +1,56 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.metrics;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotEquals;
+import static org.mockito.Mockito.mock;
+
+import org.apache.hadoop.hbase.testclassification.SmallTests;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+import com.google.common.collect.Lists;
+
+/**
+ * Test class for {@link MetricRegistriesLoader}.
+ */
+@Category(SmallTests.class)
+public class TestMetricRegistriesLoader {
+
+ @Test
+ public void testLoadSinleInstance() {
+ MetricRegistries loader = mock(MetricRegistries.class);
+ MetricRegistries instance = MetricRegistriesLoader.load(Lists.newArrayList(loader));
+ assertEquals(loader, instance);
+ }
+
+ @Test
+ public void testLoadMultipleInstances() {
+ MetricRegistries loader1 = mock(MetricRegistries.class);
+ MetricRegistries loader2 = mock(MetricRegistries.class);
+ MetricRegistries loader3 = mock(MetricRegistries.class);
+ MetricRegistries instance = MetricRegistriesLoader.load(Lists.newArrayList(loader1, loader2,
+ loader3));
+
+ // the load() returns the first instance
+ assertEquals(loader1, instance);
+ assertNotEquals(loader2, instance);
+ assertNotEquals(loader3, instance);
+ }
+}
\ No newline at end of file
diff --git hbase-metrics/README.txt hbase-metrics/README.txt
new file mode 100644
index 0000000..d80064c
--- /dev/null
+++ hbase-metrics/README.txt
@@ -0,0 +1 @@
+See the documentation at hbase-metrics-api/README.
\ No newline at end of file
diff --git hbase-metrics/pom.xml hbase-metrics/pom.xml
new file mode 100644
index 0000000..fbd2bd5
--- /dev/null
+++ hbase-metrics/pom.xml
@@ -0,0 +1,136 @@
+
+
See ExampleMasterObserverWithMetrics class in the hbase-examples modules for examples + * of how metrics can be instantiated and used.
+ * @return A MetricRegistry for the coprocessor class to track and export metrics. + */ + MetricRegistry getMetricRegistryForMaster(); } diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/MetricsCoprocessor.java hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/MetricsCoprocessor.java new file mode 100644 index 0000000..d564002 --- /dev/null +++ hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/MetricsCoprocessor.java @@ -0,0 +1,136 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.apache.hadoop.hbase.coprocessor; + +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.metrics.MetricRegistries; +import org.apache.hadoop.hbase.metrics.MetricRegistry; +import org.apache.hadoop.hbase.metrics.MetricRegistryInfo; + +import com.google.common.annotations.VisibleForTesting; + +/** + * Utility class for tracking metrics for various types of coprocessors. Each coprocessor instance + * creates its own MetricRegistry which is exported as an individual MetricSource. MetricRegistries + * are ref counted using the hbase-metric module interfaces. + */ +@InterfaceAudience.Private +public class MetricsCoprocessor { + + // Master coprocessor metrics + private static final String MASTER_COPROC_METRICS_NAME = "Coprocessor.Master"; + private static final String MASTER_COPROC_METRICS_CONTEXT = "master"; + private static final String MASTER_COPROC_METRICS_DESCRIPTION + = "Metrics about HBase MasterObservers"; + private static final String MASTER_COPROC_METRICS_JMX_CONTEXT + = "Master,sub=" + MASTER_COPROC_METRICS_NAME; + + // RegionServer coprocessor metrics + private static final String RS_COPROC_METRICS_NAME = "Coprocessor.RegionServer"; + private static final String RS_COPROC_METRICS_CONTEXT = "regionserver"; + private static final String RS_COPROC_METRICS_DESCRIPTION + = "Metrics about HBase RegionServerObservers"; + private static final String RS_COPROC_METRICS_JMX_CONTEXT = "RegionServer,sub=" + + RS_COPROC_METRICS_NAME; + + // Region coprocessor metrics + private static final String REGION_COPROC_METRICS_NAME = "Coprocessor.Region"; + private static final String REGION_COPROC_METRICS_CONTEXT = "regionserver"; + private static final String REGION_COPROC_METRICS_DESCRIPTION + = "Metrics about HBase RegionObservers"; + private static final String REGION_COPROC_METRICS_JMX_CONTEXT + = "RegionServer,sub=" + REGION_COPROC_METRICS_NAME; + + // WAL coprocessor metrics + private static final String WAL_COPROC_METRICS_NAME = "Coprocessor.WAL"; + private static final String WAL_COPROC_METRICS_CONTEXT = "regionserver"; + private static final String WAL_COPROC_METRICS_DESCRIPTION + = "Metrics about HBase WALObservers"; + private static final String WAL_COPROC_METRICS_JMX_CONTEXT + = "RegionServer,sub=" + WAL_COPROC_METRICS_NAME; + + private static String suffix(String metricName, String cpName) { + return new StringBuilder(metricName) + .append(".") + .append("CP_") + .append(cpName) + .toString(); + } + + @VisibleForTesting + static MetricRegistryInfo createRegistryInfoForMasterCoprocessor(String clazz) { + return new MetricRegistryInfo( + suffix(MASTER_COPROC_METRICS_NAME, clazz), + MASTER_COPROC_METRICS_DESCRIPTION, + suffix(MASTER_COPROC_METRICS_JMX_CONTEXT, clazz), + MASTER_COPROC_METRICS_CONTEXT, false); + } + + public static MetricRegistry createRegistryForMasterCoprocessor(String clazz) { + return MetricRegistries.global().create(createRegistryInfoForMasterCoprocessor(clazz)); + } + + @VisibleForTesting + static MetricRegistryInfo createRegistryInfoForRSCoprocessor(String clazz) { + return new MetricRegistryInfo( + suffix(RS_COPROC_METRICS_NAME, clazz), + RS_COPROC_METRICS_DESCRIPTION, + suffix(RS_COPROC_METRICS_JMX_CONTEXT, clazz), + RS_COPROC_METRICS_CONTEXT, false); + } + + public static MetricRegistry createRegistryForRSCoprocessor(String clazz) { + return MetricRegistries.global().create(createRegistryInfoForRSCoprocessor(clazz)); + } + + @VisibleForTesting + public static MetricRegistryInfo createRegistryInfoForRegionCoprocessor(String clazz) { + return new MetricRegistryInfo( + suffix(REGION_COPROC_METRICS_NAME, clazz), + REGION_COPROC_METRICS_DESCRIPTION, + suffix(REGION_COPROC_METRICS_JMX_CONTEXT, clazz), + REGION_COPROC_METRICS_CONTEXT, false); + } + + public static MetricRegistry createRegistryForRegionCoprocessor(String clazz) { + return MetricRegistries.global().create(createRegistryInfoForRegionCoprocessor(clazz)); + } + + @VisibleForTesting + public static MetricRegistryInfo createRegistryInfoForWALCoprocessor(String clazz) { + return new MetricRegistryInfo( + suffix(WAL_COPROC_METRICS_NAME, clazz), + WAL_COPROC_METRICS_DESCRIPTION, + suffix(WAL_COPROC_METRICS_JMX_CONTEXT, clazz), + WAL_COPROC_METRICS_CONTEXT, false); + } + + public static MetricRegistry createRegistryForWALCoprocessor(String clazz) { + return MetricRegistries.global().create(createRegistryInfoForWALCoprocessor(clazz)); + } + + public static void removeRegistry(MetricRegistry registry) { + if (registry == null) { + return; + } + MetricRegistries.global().remove(registry.getMetricRegistryInfo()); + } +} diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionCoprocessorEnvironment.java hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionCoprocessorEnvironment.java index bdf88af..3566f06 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionCoprocessorEnvironment.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionCoprocessorEnvironment.java @@ -21,11 +21,12 @@ package org.apache.hadoop.hbase.coprocessor; import java.util.concurrent.ConcurrentMap; -import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.hbase.classification.InterfaceStability; import org.apache.hadoop.hbase.CoprocessorEnvironment; import org.apache.hadoop.hbase.HBaseInterfaceAudience; import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.classification.InterfaceStability; +import org.apache.hadoop.hbase.metrics.MetricRegistry; import org.apache.hadoop.hbase.regionserver.Region; import org.apache.hadoop.hbase.regionserver.RegionServerServices; @@ -43,4 +44,23 @@ public interface RegionCoprocessorEnvironment extends CoprocessorEnvironment { /** @return shared data between all instances of this coprocessor */ ConcurrentMapSee ExampleRegionObserverWithMetrics class in the hbase-examples modules to see examples of how + * metrics can be instantiated and used.
+ * @return A MetricRegistry for the coprocessor class to track and export metrics. + */ + // Note: we are not exposing getMetricRegistryForRegion(). per-region metrics are already costly + // so we do not want to allow coprocessors to export metrics at the region level. We can allow + // getMetricRegistryForTable() to allow coprocessors to track metrics per-table, per-regionserver. + MetricRegistry getMetricRegistryForRegionServer(); + + } diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionServerCoprocessorEnvironment.java hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionServerCoprocessorEnvironment.java index 527df45..f42556a 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionServerCoprocessorEnvironment.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionServerCoprocessorEnvironment.java @@ -19,6 +19,7 @@ package org.apache.hadoop.hbase.coprocessor; import org.apache.hadoop.hbase.CoprocessorEnvironment; +import org.apache.hadoop.hbase.metrics.MetricRegistry; import org.apache.hadoop.hbase.regionserver.RegionServerServices; public interface RegionServerCoprocessorEnvironment extends CoprocessorEnvironment { @@ -28,4 +29,13 @@ public interface RegionServerCoprocessorEnvironment extends CoprocessorEnvironme * @return the region server services */ RegionServerServices getRegionServerServices(); + + /** + * Returns a MetricRegistry that can be used to track metrics at the region server level. + * + *See ExampleMasterObserverWithMetrics class in the hbase-examples modules for examples + * of how metrics can be instantiated and used.
+ * @return A MetricRegistry for the coprocessor class to track and export metrics. + */ + MetricRegistry getMetricRegistryForRegionServer(); } diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/WALCoprocessorEnvironment.java hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/WALCoprocessorEnvironment.java index a4ce5f1..0865d96 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/WALCoprocessorEnvironment.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/WALCoprocessorEnvironment.java @@ -23,6 +23,7 @@ import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; import org.apache.hadoop.hbase.CoprocessorEnvironment; import org.apache.hadoop.hbase.HBaseInterfaceAudience; +import org.apache.hadoop.hbase.metrics.MetricRegistry; import org.apache.hadoop.hbase.wal.WAL; @InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.COPROC) @@ -30,4 +31,13 @@ import org.apache.hadoop.hbase.wal.WAL; public interface WALCoprocessorEnvironment extends CoprocessorEnvironment { /** @return reference to the region server's WAL */ WAL getWAL(); + + /** + * Returns a MetricRegistry that can be used to track metrics at the region server level. + * + *See ExampleRegionServerObserverWithMetrics class in the hbase-examples modules for examples + * of how metrics can be instantiated and used.
+ * @return A MetricRegistry for the coprocessor class to track and export metrics. + */ + MetricRegistry getMetricRegistryForRegionServer(); } diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/AgeSnapshot.java hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/AgeSnapshot.java index 4c1ad23..dd3bf25 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/AgeSnapshot.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/AgeSnapshot.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.hbase.io.hfile; -import org.apache.hadoop.hbase.util.FastLongHistogram; +import org.apache.hadoop.hbase.metrics.impl.FastLongHistogram; import org.codehaus.jackson.annotate.JsonIgnoreProperties; /** diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCacheUtil.java hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCacheUtil.java index 3c11149..9335ef6 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCacheUtil.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCacheUtil.java @@ -25,7 +25,7 @@ import java.util.concurrent.ConcurrentSkipListSet; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.util.FastLongHistogram; +import org.apache.hadoop.hbase.metrics.impl.FastLongHistogram; import org.codehaus.jackson.JsonGenerationException; import org.codehaus.jackson.annotate.JsonIgnoreProperties; import org.codehaus.jackson.map.JsonMappingException; diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheStats.java hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheStats.java index 7264eda..76e6d2a 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheStats.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheStats.java @@ -23,7 +23,7 @@ import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.LongAdder; import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.hbase.util.FastLongHistogram; +import org.apache.hadoop.hbase.metrics.impl.FastLongHistogram; /** diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java index 3dec2e8..28a8550 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java @@ -19,8 +19,6 @@ package org.apache.hadoop.hbase.master; -import com.google.common.net.HostAndPort; - import java.io.IOException; import java.util.List; import java.util.Set; @@ -45,15 +43,19 @@ import org.apache.hadoop.hbase.coprocessor.CoprocessorHost; import org.apache.hadoop.hbase.coprocessor.CoprocessorService; import org.apache.hadoop.hbase.coprocessor.MasterCoprocessorEnvironment; import org.apache.hadoop.hbase.coprocessor.MasterObserver; +import org.apache.hadoop.hbase.coprocessor.MetricsCoprocessor; import org.apache.hadoop.hbase.coprocessor.ObserverContext; import org.apache.hadoop.hbase.ipc.RpcServer; import org.apache.hadoop.hbase.master.locking.LockProcedure; import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv; +import org.apache.hadoop.hbase.metrics.MetricRegistry; import org.apache.hadoop.hbase.procedure2.ProcedureExecutor; import org.apache.hadoop.hbase.replication.ReplicationPeerConfig; +import org.apache.hadoop.hbase.security.User; import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.SnapshotDescription; import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.Quotas; -import org.apache.hadoop.hbase.security.User; + +import com.google.common.net.HostAndPort; /** * Provides the coprocessor framework and environment for master oriented @@ -72,8 +74,9 @@ public class MasterCoprocessorHost */ static class MasterEnvironment extends CoprocessorHost.Environment implements MasterCoprocessorEnvironment { - private MasterServices masterServices; - final boolean supportGroupCPs; + private final MasterServices masterServices; + private final boolean supportGroupCPs; + private final MetricRegistry metricRegistry; public MasterEnvironment(final Class> implClass, final Coprocessor impl, final int priority, final int seq, final Configuration conf, @@ -82,11 +85,25 @@ public class MasterCoprocessorHost this.masterServices = services; supportGroupCPs = !useLegacyMethod(impl.getClass(), "preBalanceRSGroup", ObserverContext.class, String.class); + this.metricRegistry = + MetricsCoprocessor.createRegistryForMasterCoprocessor(implClass.getName()); } + @Override public MasterServices getMasterServices() { return masterServices; } + + @Override + public MetricRegistry getMetricRegistryForMaster() { + return metricRegistry; + } + + @Override + protected void shutdown() { + super.shutdown(); + MetricsCoprocessor.removeRegistry(this.metricRegistry); + } } private MasterServices masterServices; diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServer.java hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServer.java index 8bca6c5..4c28763 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServer.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServer.java @@ -17,10 +17,14 @@ */ package org.apache.hadoop.hbase.regionserver; -import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.hbase.CompatibilitySingletonFactory; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; -import org.apache.hadoop.hbase.CompatibilitySingletonFactory; +import org.apache.hadoop.hbase.metrics.MetricRegistries; +import org.apache.hadoop.hbase.metrics.MetricRegistry; +import org.apache.hadoop.hbase.metrics.Timer; + +import com.google.common.annotations.VisibleForTesting; /** *
@@ -36,11 +40,20 @@ public class MetricsRegionServer {
private MetricsRegionServerSource serverSource;
private MetricsRegionServerWrapper regionServerWrapper;
+ private MetricRegistry metricRegistry;
+ private Timer bulkLoadTimer;
+
public MetricsRegionServer(MetricsRegionServerWrapper regionServerWrapper) {
this(regionServerWrapper,
CompatibilitySingletonFactory.getInstance(MetricsRegionServerSourceFactory.class)
.createServer(regionServerWrapper));
+ // Create hbase-metrics module based metrics. The registry should already be registered by the
+ // MetricsRegionServerSource
+ metricRegistry = MetricRegistries.global().get(serverSource.getMetricRegistryInfo()).get();
+
+ // create and use metrics from the new hbase-metrics based registry.
+ bulkLoadTimer = metricRegistry.timer("Bulkload");
}
MetricsRegionServer(MetricsRegionServerWrapper regionServerWrapper,
@@ -131,4 +144,8 @@ public class MetricsRegionServer {
serverSource.updateCompactionInputSize(isMajor, inputBytes);
serverSource.updateCompactionOutputSize(isMajor, outputBytes);
}
+
+ public void updateBulkLoad(long millis) {
+ this.bulkLoadTimer.updateMillis(millis);
+ }
}
diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java
index a072dce..3e827aa 100644
--- hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java
+++ hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java
@@ -208,6 +208,8 @@ import org.apache.hadoop.hbase.wal.WALSplitter;
import org.apache.hadoop.hbase.zookeeper.ZKSplitLog;
import org.apache.zookeeper.KeeperException;
+import com.google.common.annotations.VisibleForTesting;
+
/**
* Implements the regionserver RPC services.
*/
@@ -2108,6 +2110,7 @@ public class RSRpcServices implements HBaseRPCErrorHandler,
@Override
public BulkLoadHFileResponse bulkLoadHFile(final RpcController controller,
final BulkLoadHFileRequest request) throws ServiceException {
+ long start = EnvironmentEdgeManager.currentTime();
try {
checkOpen();
requestCount.increment();
@@ -2152,6 +2155,11 @@ public class RSRpcServices implements HBaseRPCErrorHandler,
return builder.build();
} catch (IOException ie) {
throw new ServiceException(ie);
+ } finally {
+ if (regionServer.metricsRegionServer != null) {
+ regionServer.metricsRegionServer.updateBulkLoad(
+ EnvironmentEdgeManager.currentTime() - start);
+ }
}
}
diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java
index 94508b6..e4b47bc 100644
--- hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java
+++ hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java
@@ -19,11 +19,6 @@
package org.apache.hadoop.hbase.regionserver;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.Lists;
-import com.google.protobuf.Message;
-import com.google.protobuf.Service;
-
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
@@ -44,7 +39,6 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.Coprocessor;
-import org.apache.hadoop.hbase.CoprocessorEnvironment;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HBaseInterfaceAudience;
import org.apache.hadoop.hbase.HConstants;
@@ -65,6 +59,7 @@ import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver;
import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
import org.apache.hadoop.hbase.coprocessor.CoprocessorService;
import org.apache.hadoop.hbase.coprocessor.EndpointObserver;
+import org.apache.hadoop.hbase.coprocessor.MetricsCoprocessor;
import org.apache.hadoop.hbase.coprocessor.ObserverContext;
import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
import org.apache.hadoop.hbase.coprocessor.RegionObserver;
@@ -75,6 +70,7 @@ import org.apache.hadoop.hbase.io.FSDataInputStreamWrapper;
import org.apache.hadoop.hbase.io.Reference;
import org.apache.hadoop.hbase.io.hfile.CacheConfig;
import org.apache.hadoop.hbase.ipc.RpcServer;
+import org.apache.hadoop.hbase.metrics.MetricRegistry;
import org.apache.hadoop.hbase.regionserver.Region.Operation;
import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest;
import org.apache.hadoop.hbase.regionserver.querymatcher.DeleteTracker;
@@ -85,6 +81,11 @@ import org.apache.hadoop.hbase.util.CoprocessorClassLoader;
import org.apache.hadoop.hbase.util.Pair;
import org.apache.hadoop.hbase.wal.WALKey;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
+import com.google.protobuf.Message;
+import com.google.protobuf.Service;
+
/**
* Implements the coprocessor environment and runtime support for coprocessors
* loaded within a {@link Region}.
@@ -112,6 +113,7 @@ public class RegionCoprocessorHost
private Region region;
private RegionServerServices rsServices;
ConcurrentMap