diff --git a/common/pom.xml b/common/pom.xml index 1b9b4bc..67aab7c 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -188,6 +188,11 @@ jackson-databind ${jackson.new.version} + + com.github.joshelser + dropwizard-metrics-hadoop-metrics2-reporter + ${dropwizard-metrics-hadoop-metrics2-reporter.version} + diff --git a/common/src/java/org/apache/hadoop/hive/common/metrics/metrics2/CodahaleMetrics.java b/common/src/java/org/apache/hadoop/hive/common/metrics/metrics2/CodahaleMetrics.java index 7dea8de..4c43367 100644 --- a/common/src/java/org/apache/hadoop/hive/common/metrics/metrics2/CodahaleMetrics.java +++ b/common/src/java/org/apache/hadoop/hive/common/metrics/metrics2/CodahaleMetrics.java @@ -34,6 +34,7 @@ import com.codahale.metrics.jvm.MemoryUsageGaugeSet; import com.codahale.metrics.jvm.ThreadStatesGaugeSet; import com.fasterxml.jackson.databind.ObjectMapper; +import com.github.joshelser.dropwizard.metrics.hadoop.HadoopMetrics2Reporter; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Splitter; import com.google.common.cache.CacheBuilder; @@ -47,6 +48,7 @@ import org.apache.hadoop.hive.common.metrics.common.MetricsScope; import org.apache.hadoop.hive.common.metrics.common.MetricsVariable; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -372,6 +374,20 @@ private void initReporting(Set reportingSet) throws Exception jsonFileReporter.start(); reporters.add(jsonFileReporter); break; + case HADOOP2: + String applicationName = conf.get(HiveConf.ConfVars.HIVE_METRICS_HADOOP2_COMPONENT_NAME.varname); + long reportingInterval = HiveConf.toTime( + conf.get(HiveConf.ConfVars.HIVE_METRICS_HADOOP2_INTERVAL.varname), + TimeUnit.SECONDS, TimeUnit.SECONDS); + final HadoopMetrics2Reporter metrics2Reporter = HadoopMetrics2Reporter.forRegistry(metricRegistry) + .convertRatesTo(TimeUnit.SECONDS) + .convertDurationsTo(TimeUnit.MILLISECONDS) + .build(DefaultMetricsSystem.initialize(applicationName), // The application-level name + applicationName, // Component name + applicationName, // Component description + "General"); // Name for each metric record + metrics2Reporter.start(reportingInterval, TimeUnit.SECONDS); + break; } } } diff --git a/common/src/java/org/apache/hadoop/hive/common/metrics/metrics2/MetricsReporting.java b/common/src/java/org/apache/hadoop/hive/common/metrics/metrics2/MetricsReporting.java index 643246f..e6d1f1b 100644 --- a/common/src/java/org/apache/hadoop/hive/common/metrics/metrics2/MetricsReporting.java +++ b/common/src/java/org/apache/hadoop/hive/common/metrics/metrics2/MetricsReporting.java @@ -23,5 +23,6 @@ public enum MetricsReporting { JMX, CONSOLE, - JSON_FILE + JSON_FILE, + HADOOP2 } diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 95c5c0e..06d035c 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -1960,7 +1960,8 @@ private static void populateLlapDaemonVarsSet(Set llapDaemonVarsSetLocal "org.apache.hadoop.hive.common.metrics.LegacyMetrics"), "Hive metrics subsystem implementation class."), HIVE_METRICS_REPORTER("hive.service.metrics.reporter", "JSON_FILE, JMX", - "Reporter type for metric class org.apache.hadoop.hive.common.metrics.metrics2.CodahaleMetrics, comma separated list of JMX, CONSOLE, JSON_FILE"), + "Reporter type for metric class org.apache.hadoop.hive.common.metrics.metrics2.CodahaleMetrics, " + + "comma separated list of JMX, CONSOLE, JSON_FILE, HADOOP2"), HIVE_METRICS_JSON_FILE_LOCATION("hive.service.metrics.file.location", "/tmp/report.json", "For metric class org.apache.hadoop.hive.common.metrics.metrics2.CodahaleMetrics JSON_FILE reporter, the location of local JSON metrics file. " + "This file will get overwritten at every interval."), @@ -1968,6 +1969,15 @@ private static void populateLlapDaemonVarsSet(Set llapDaemonVarsSetLocal new TimeValidator(TimeUnit.MILLISECONDS), "For metric class org.apache.hadoop.hive.common.metrics.metrics2.CodahaleMetrics JSON_FILE reporter, " + "the frequency of updating JSON metrics file."), + HIVE_METRICS_HADOOP2_INTERVAL("hive.service.metrics.hadoop2.frequency", "30s", + new TimeValidator(TimeUnit.SECONDS), + "For metric class org.apache.hadoop.hive.common.metrics.metrics2.CodahaleMetrics HADOOP2 reporter, " + + "the frequency of updating the HADOOP2 metrics system."), + HIVE_METRICS_HADOOP2_COMPONENT_NAME("hive.service.metrics.hadoop2.component", + "hive", + "Component name to provide to Hadoop2 Metrics system. Ideally 'hivemetastore' for the MetaStore " + + " and and 'hiveserver2' for HiveServer2." + ), HIVE_PERF_LOGGER("hive.exec.perf.logger", "org.apache.hadoop.hive.ql.log.PerfLogger", "The class responsible for logging client side performance metrics. \n" + "Must be a subclass of org.apache.hadoop.hive.ql.log.PerfLogger"), diff --git a/pom.xml b/pom.xml index 2337e89..540fbb7 100644 --- a/pom.xml +++ b/pom.xml @@ -128,6 +128,7 @@ 1.4 10.10.2.0 3.1.0 + 0.1.0 14.0.1 2.4.4 2.6.0