diff --git a/common/src/java/org/apache/hadoop/hive/common/metrics/LegacyMetrics.java b/common/src/java/org/apache/hadoop/hive/common/metrics/LegacyMetrics.java index 0f082f683afc6070dbdff5fd269e3dde96befe55..090db3d09d2084af11e54c97e7e7a398fded9ad9 100644 --- a/common/src/java/org/apache/hadoop/hive/common/metrics/LegacyMetrics.java +++ b/common/src/java/org/apache/hadoop/hive/common/metrics/LegacyMetrics.java @@ -231,6 +231,10 @@ public void addRatio(String name, MetricsVariable numerator, //Not implemented } + public void markMeter(String name) { + //Not implemented. + } + public void set(String name, Object value) { metrics.put(name,value); } diff --git a/common/src/java/org/apache/hadoop/hive/common/metrics/common/Metrics.java b/common/src/java/org/apache/hadoop/hive/common/metrics/common/Metrics.java index 8fb7c5ad099474de04913a4e13517a7e59414072..368dd4f12bb3875aca23efd80c571a108427eb10 100644 --- a/common/src/java/org/apache/hadoop/hive/common/metrics/common/Metrics.java +++ b/common/src/java/org/apache/hadoop/hive/common/metrics/common/Metrics.java @@ -103,4 +103,10 @@ public void addRatio(String name, MetricsVariable numerator, MetricsVariable denominator); + /** + * Mark an event occurance for a meter. Meters measure the rate of an event and track + * 1/5/15 minute moving averages + * @param name name of the meter + */ + public void markMeter(String name); } diff --git a/common/src/java/org/apache/hadoop/hive/common/metrics/common/MetricsConstant.java b/common/src/java/org/apache/hadoop/hive/common/metrics/common/MetricsConstant.java index b4a7dcc147580a94b6bfe2e729385ace37ae1436..4c53297bd7720ecde7f87bea1f06c9dda4d2c4bd 100644 --- a/common/src/java/org/apache/hadoop/hive/common/metrics/common/MetricsConstant.java +++ b/common/src/java/org/apache/hadoop/hive/common/metrics/common/MetricsConstant.java @@ -22,6 +22,9 @@ */ public class MetricsConstant { + public static final String API_PREFIX = "api_"; + public static final String ACTIVE_CALLS = "active_calls_"; + public static final String JVM_PAUSE_INFO = "jvm.pause.info-threshold"; public static final String JVM_PAUSE_WARN = "jvm.pause.warn-threshold"; public static final String JVM_EXTRA_SLEEP = "jvm.pause.extraSleepTime"; @@ -73,4 +76,9 @@ public static final String HS2_AVG_OPEN_SESSION_TIME = "hs2_avg_open_session_time"; public static final String HS2_AVG_ACTIVE_SESSION_TIME = "hs2_avg_active_session_time"; + public static final String HS2_SUBMITTED_QURIES = "hs2_submitted_queries"; + public static final String HS2_COMPILING_QUERIES = "hs2_compiling_queries"; + public static final String HS2_EXECUTING_QUERIES = "hs2_executing_queries"; + public static final String HS2_FAILED_QUERIES = "hs2_failed_queries"; + public static final String HS2_SUCEEDED_QUERIES = "hs2_suceeded_queries"; } \ No newline at end of file diff --git a/common/src/java/org/apache/hadoop/hive/common/metrics/metrics2/CodahaleMetrics.java b/common/src/java/org/apache/hadoop/hive/common/metrics/metrics2/CodahaleMetrics.java index cd3d627be65712c3d0556e7e0a34bfe9602318fa..e8abf6cf06afc9fa590af3a447eacc67735a69e6 100644 --- a/common/src/java/org/apache/hadoop/hive/common/metrics/metrics2/CodahaleMetrics.java +++ b/common/src/java/org/apache/hadoop/hive/common/metrics/metrics2/CodahaleMetrics.java @@ -23,6 +23,7 @@ import com.codahale.metrics.ExponentiallyDecayingReservoir; import com.codahale.metrics.Gauge; import com.codahale.metrics.JmxReporter; +import com.codahale.metrics.Meter; import com.codahale.metrics.Metric; import com.codahale.metrics.MetricRegistry; import com.codahale.metrics.MetricSet; @@ -46,6 +47,7 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.hive.common.metrics.common.MetricsConstant; import org.apache.hadoop.hive.common.metrics.common.MetricsScope; import org.apache.hadoop.hive.common.metrics.common.MetricsVariable; import org.apache.hadoop.hive.conf.HiveConf; @@ -76,17 +78,17 @@ */ public class CodahaleMetrics implements org.apache.hadoop.hive.common.metrics.common.Metrics { - public static final String API_PREFIX = "api_"; - public static final String ACTIVE_CALLS = "active_calls_"; public static final Logger LOGGER = LoggerFactory.getLogger(CodahaleMetrics.class); public final MetricRegistry metricRegistry = new MetricRegistry(); private final Lock timersLock = new ReentrantLock(); private final Lock countersLock = new ReentrantLock(); private final Lock gaugesLock = new ReentrantLock(); + private final Lock metersLock = new ReentrantLock(); private LoadingCache timers; private LoadingCache counters; + private LoadingCache meters; private ConcurrentHashMap gauges; private HiveConf conf; @@ -126,7 +128,7 @@ public void open() { if (!isOpen) { isOpen = true; this.timerContext = timer.time(); - CodahaleMetrics.this.incrementCounter(ACTIVE_CALLS + name); + CodahaleMetrics.this.incrementCounter(MetricsConstant.ACTIVE_CALLS + name); } else { LOGGER.warn("Scope named " + name + " is not closed, cannot be opened."); } @@ -138,7 +140,7 @@ public void open() { public void close() { if (isOpen) { timerContext.close(); - CodahaleMetrics.this.decrementCounter(ACTIVE_CALLS + name); + CodahaleMetrics.this.decrementCounter(MetricsConstant.ACTIVE_CALLS + name); } else { LOGGER.warn("Scope named " + name + " is not open, cannot be closed."); } @@ -169,6 +171,16 @@ public Counter load(String key) { } } ); + meters = CacheBuilder.newBuilder().build( + new CacheLoader() { + @Override + public Meter load(String key) { + Meter meter = new Meter(); + metricRegistry.register(key, meter); + return meter; + } + } + ); gauges = new ConcurrentHashMap(); //register JVM metrics @@ -209,11 +221,11 @@ public void close() throws Exception { } timers.invalidateAll(); counters.invalidateAll(); + meters.invalidateAll(); } @Override public void startStoredScope(String name) { - name = API_PREFIX + name; if (threadLocalScopes.get().containsKey(name)) { threadLocalScopes.get().get(name).open(); } else { @@ -223,7 +235,6 @@ public void startStoredScope(String name) { @Override public void endStoredScope(String name) { - name = API_PREFIX + name; if (threadLocalScopes.get().containsKey(name)) { threadLocalScopes.get().get(name).close(); threadLocalScopes.get().remove(name); @@ -239,7 +250,6 @@ public MetricsScope getStoredScope(String name) throws IllegalArgumentException } public MetricsScope createScope(String name) { - name = API_PREFIX + name; return new CodahaleMetricsScope(name); } @@ -322,6 +332,21 @@ private void addGaugeInternal(String name, Gauge gauge) { } } + @Override + public void markMeter(String name) { + String key = name; + try { + metersLock.lock(); + Meter meter = meters.get(name); + meter.mark(); + } catch (ExecutionException e) { + throw new IllegalStateException("Error retrieving meter " + name + + " from the metric registry ", e); + } finally { + metersLock.unlock(); + } + } + // This method is necessary to synchronize lazy-creation to the timers. private Timer getTimer(String name) { String key = name; @@ -330,7 +355,8 @@ private Timer getTimer(String name) { Timer timer = timers.get(key); return timer; } catch (ExecutionException e) { - throw new IllegalStateException("Error retrieving timer from the metric registry ", e); + throw new IllegalStateException("Error retrieving timer " + name + + " from the metric registry ", e); } finally { timersLock.unlock(); } diff --git a/common/src/java/org/apache/hadoop/hive/ql/log/PerfLogger.java b/common/src/java/org/apache/hadoop/hive/ql/log/PerfLogger.java index 7658f1c3e0ae7112d10aaf195197ed7e0d318351..7f3c8b312d8a815f5b255683bbb503dc6c158d09 100644 --- a/common/src/java/org/apache/hadoop/hive/ql/log/PerfLogger.java +++ b/common/src/java/org/apache/hadoop/hive/ql/log/PerfLogger.java @@ -20,6 +20,7 @@ import com.google.common.collect.ImmutableMap; import org.apache.hadoop.hive.common.metrics.common.Metrics; +import org.apache.hadoop.hive.common.metrics.common.MetricsConstant; import org.apache.hadoop.hive.common.metrics.common.MetricsFactory; import org.apache.hadoop.hive.common.metrics.common.MetricsScope; import org.apache.hadoop.hive.conf.HiveConf; @@ -27,13 +28,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; -import java.util.Collection; import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; import java.util.Map; -import java.util.Set; /** * PerfLogger. @@ -225,7 +221,7 @@ public Long getDuration(String method) { private void beginMetrics(String method) { Metrics metrics = MetricsFactory.getInstance(); if (metrics != null) { - MetricsScope scope = metrics.createScope(method); + MetricsScope scope = metrics.createScope(MetricsConstant.API_PREFIX + method); openScopes.put(method, scope); } diff --git a/common/src/test/org/apache/hadoop/hive/common/metrics/MetricsTestUtils.java b/common/src/test/org/apache/hadoop/hive/common/metrics/MetricsTestUtils.java index 3bb7a1e2451695859e7957697f84eb17c4dabff1..5c38780be90bfe8f7cb8e808a3abcbfa206aa2cd 100644 --- a/common/src/test/org/apache/hadoop/hive/common/metrics/MetricsTestUtils.java +++ b/common/src/test/org/apache/hadoop/hive/common/metrics/MetricsTestUtils.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hive.common.metrics; +import com.codahale.metrics.Meter; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import org.junit.Assert; @@ -33,6 +34,7 @@ public static final MetricsCategory COUNTER = new MetricsCategory("counters", "count"); public static final MetricsCategory TIMER = new MetricsCategory("timers", "count"); public static final MetricsCategory GAUGE = new MetricsCategory("gauges", "value"); + public static final MetricsCategory METER = new MetricsCategory("meters", "count"); static class MetricsCategory { String category; diff --git a/common/src/test/org/apache/hadoop/hive/common/metrics/metrics2/TestCodahaleMetrics.java b/common/src/test/org/apache/hadoop/hive/common/metrics/metrics2/TestCodahaleMetrics.java index 6ee6245c1212c06c2ca9cc7a795f288c3928d675..aa4e75f9f8160d1b54b14c1a23ea42e156bd45ca 100644 --- a/common/src/test/org/apache/hadoop/hive/common/metrics/metrics2/TestCodahaleMetrics.java +++ b/common/src/test/org/apache/hadoop/hive/common/metrics/metrics2/TestCodahaleMetrics.java @@ -80,7 +80,7 @@ public void testScope() throws Exception { MetricsFactory.getInstance().endStoredScope("method1"); } - Timer timer = metricRegistry.getTimers().get("api_method1"); + Timer timer = metricRegistry.getTimers().get("method1"); Assert.assertEquals(5, timer.getCount()); Assert.assertTrue(timer.getMeanRate() > 0); } @@ -113,7 +113,7 @@ public Void call() throws Exception { } executorService.shutdown(); assertTrue(executorService.awaitTermination(10000, TimeUnit.MILLISECONDS)); - Timer timer = metricRegistry.getTimers().get("api_method2"); + Timer timer = metricRegistry.getTimers().get("method2"); Assert.assertEquals(4, timer.getCount()); Assert.assertTrue(timer.getMeanRate() > 0); } @@ -161,4 +161,20 @@ public void testGauge() throws Exception { json = ((CodahaleMetrics) MetricsFactory.getInstance()).dumpJson(); MetricsTestUtils.verifyMetricsJson(json, MetricsTestUtils.GAUGE, "gauge1", testVar.getValue()); } + + @Test + public void testMeter() throws Exception { + + String json = ((CodahaleMetrics) MetricsFactory.getInstance()).dumpJson(); + MetricsTestUtils.verifyMetricsJson(json, MetricsTestUtils.METER, "meter", ""); + + MetricsFactory.getInstance().markMeter("meter"); + json = ((CodahaleMetrics) MetricsFactory.getInstance()).dumpJson(); + MetricsTestUtils.verifyMetricsJson(json, MetricsTestUtils.METER, "meter", "1"); + + MetricsFactory.getInstance().markMeter("meter"); + json = ((CodahaleMetrics) MetricsFactory.getInstance()).dumpJson(); + MetricsTestUtils.verifyMetricsJson(json, MetricsTestUtils.METER, "meter", "2"); + + } } diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java index 13d0aab40eaf344869b8230614b55115f64857e6..d0e8496f59b92c9e85103376535c6f21745d8788 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java @@ -780,7 +780,7 @@ private String startFunction(String function, String extraLogInfo) { logInfo((getThreadLocalIpAddress() == null ? "" : "source:" + getThreadLocalIpAddress() + " ") + function + extraLogInfo); if (MetricsFactory.getInstance() != null) { - MetricsFactory.getInstance().startStoredScope(function); + MetricsFactory.getInstance().startStoredScope(MetricsConstant.API_PREFIX + function); } return function; } @@ -819,7 +819,7 @@ private void endFunction(String function, boolean successful, Exception e, private void endFunction(String function, MetaStoreEndFunctionContext context) { if (MetricsFactory.getInstance() != null) { - MetricsFactory.getInstance().endStoredScope(function); + MetricsFactory.getInstance().endStoredScope(MetricsConstant.API_PREFIX + function); } for (MetaStoreEndFunctionListener listener : endFunctionListeners) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java index efa2bdc81c7c77c242b66f8f0d3708e7cd245c8a..2719cb8f321921d40b655384b5a3b103461618d7 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java @@ -37,6 +37,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.ReentrantLock; +import com.google.common.collect.Iterables; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.hive.common.ValidTxnList; @@ -65,6 +66,7 @@ import org.apache.hadoop.hive.ql.hooks.Hook; import org.apache.hadoop.hive.ql.hooks.HookContext; import org.apache.hadoop.hive.ql.hooks.HookUtils; +import org.apache.hadoop.hive.ql.hooks.MetricsQueryLifeTimeHook; import org.apache.hadoop.hive.ql.hooks.PostExecute; import org.apache.hadoop.hive.ql.hooks.PreExecute; import org.apache.hadoop.hive.ql.hooks.QueryLifeTimeHook; @@ -88,7 +90,6 @@ import org.apache.hadoop.hive.ql.parse.ASTNode; import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer; import org.apache.hadoop.hive.ql.parse.ColumnAccessInfo; -import org.apache.hadoop.hive.ql.parse.ExplainSemanticAnalyzer; import org.apache.hadoop.hive.ql.parse.HiveSemanticAnalyzerHook; import org.apache.hadoop.hive.ql.parse.HiveSemanticAnalyzerHookContext; import org.apache.hadoop.hive.ql.parse.HiveSemanticAnalyzerHookContextImpl; @@ -178,7 +179,7 @@ private QueryState queryState; // Query hooks that execute before compilation and after execution - List queryHooks; + private List queryHooks; public enum DriverState { INITIALIZED, @@ -432,8 +433,8 @@ public int compile(String command, boolean resetTaskIds, boolean deferClose) { // Whether any error occurred during query compilation. Used for query lifetime hook. boolean compileError = false; - try { + // Initialize the transaction manager. This must be done before analyze is called. final HiveTxnManager txnManager = SessionState.get().initTxnMgr(conf); // In case when user Ctrl-C twice to kill Hive CLI JVM, we want to release locks @@ -471,7 +472,7 @@ public void run() { perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.PARSE); // Trigger query hook before compilation - queryHooks = getHooks(ConfVars.HIVE_QUERY_LIFETIME_HOOKS, QueryLifeTimeHook.class); + queryHooks = loadQueryHooks(); if (queryHooks != null && !queryHooks.isEmpty()) { QueryLifeTimeHookContext qhc = new QueryLifeTimeHookContextImpl(); qhc.setHiveConf(conf); @@ -666,6 +667,19 @@ private boolean isInterrupted() { } } + private List loadQueryHooks() throws Exception { + List hooks = new ArrayList<>(); + + if (conf.getBoolVar(ConfVars.HIVE_SERVER2_METRICS_ENABLED)) { + hooks.add(new MetricsQueryLifeTimeHook()); + } + List propertyDefinedHoooks = getHooks(ConfVars.HIVE_QUERY_LIFETIME_HOOKS, QueryLifeTimeHook.class); + if (propertyDefinedHoooks != null) { + Iterables.addAll(hooks, propertyDefinedHoooks); + } + return hooks; + } + private ImmutableMap dumpMetaCallTimingWithoutEx(String phase) { try { return Hive.get().dumpAndClearMetaCallTiming(phase); @@ -1687,6 +1701,7 @@ public int execute(boolean deferClose) throws CommandNeedRetryException { boolean noName = StringUtils.isEmpty(conf.get(MRJobConfig.JOB_NAME)); int maxlen = conf.getIntVar(HiveConf.ConfVars.HIVEJOBNAMELENGTH); + Metrics metrics = MetricsFactory.getInstance(); String queryId = conf.getVar(HiveConf.ConfVars.HIVEQUERYID); // Get the query string from the conf file as the compileInternal() method might @@ -1811,7 +1826,6 @@ public int execute(boolean deferClose) throws CommandNeedRetryException { assert tsk.getParentTasks() == null || tsk.getParentTasks().isEmpty(); driverCxt.addToRunnable(tsk); - Metrics metrics = MetricsFactory.getInstance(); if (metrics != null) { tsk.updateTaskMetrics(metrics); } @@ -2435,7 +2449,7 @@ public void setOperationId(String opId) { this.operationId = opId; } - /** + /** * Resets QueryState to get new queryId on Driver reuse. */ public void resetQueryState() { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/hooks/MetricsQueryLifeTimeHook.java b/ql/src/java/org/apache/hadoop/hive/ql/hooks/MetricsQueryLifeTimeHook.java new file mode 100644 index 0000000000000000000000000000000000000000..246ce0e1d9a9068277fcfe33f2b6c76c1b7c61fa --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/hooks/MetricsQueryLifeTimeHook.java @@ -0,0 +1,62 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.hooks; + +import org.apache.hadoop.hive.common.metrics.common.Metrics; +import org.apache.hadoop.hive.common.metrics.common.MetricsConstant; +import org.apache.hadoop.hive.common.metrics.common.MetricsFactory; +import org.apache.hadoop.hive.common.metrics.common.MetricsScope; + +/** + * LifeTimeHook gathering metrics for the query lifecycle if the + * metrics are enabled + */ +public class MetricsQueryLifeTimeHook implements QueryLifeTimeHook { + + private Metrics metrics = MetricsFactory.getInstance(); + private MetricsScope compilingQryScp; + private MetricsScope executingQryScp; + + @Override + public void beforeCompile(QueryLifeTimeHookContext ctx) { + if (metrics != null) { + compilingQryScp = metrics.createScope(MetricsConstant.HS2_COMPILING_QUERIES); + } + } + + @Override + public void afterCompile(QueryLifeTimeHookContext ctx, boolean hasError) { + if (metrics != null && compilingQryScp != null) { + metrics.endScope(compilingQryScp); + } + } + + @Override + public void beforeExecution(QueryLifeTimeHookContext ctx) { + if (metrics != null) { + executingQryScp = metrics.createScope(MetricsConstant.HS2_EXECUTING_QUERIES); + } + } + + @Override + public void afterExecution(QueryLifeTimeHookContext ctx, boolean hasError) { + if (metrics != null && executingQryScp != null) { + metrics.endScope(executingQryScp); + } + } +} diff --git a/ql/src/test/org/apache/hadoop/hive/ql/hooks/TestMetricsQueryLifeTimeHook.java b/ql/src/test/org/apache/hadoop/hive/ql/hooks/TestMetricsQueryLifeTimeHook.java new file mode 100644 index 0000000000000000000000000000000000000000..0fb7e3e9287ca77f4b06d8eedb1b0e3b9919597e --- /dev/null +++ b/ql/src/test/org/apache/hadoop/hive/ql/hooks/TestMetricsQueryLifeTimeHook.java @@ -0,0 +1,117 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.hooks; + +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.nullValue; +import static org.junit.Assert.assertThat; + +import com.codahale.metrics.Counter; +import com.codahale.metrics.MetricRegistry; +import com.codahale.metrics.Timer; +import org.apache.hadoop.fs.CommonConfigurationKeysPublic; +import org.apache.hadoop.hive.common.metrics.common.MetricsConstant; +import org.apache.hadoop.hive.common.metrics.common.MetricsFactory; +import org.apache.hadoop.hive.common.metrics.metrics2.CodahaleMetrics; +import org.apache.hadoop.hive.common.metrics.metrics2.MetricsReporting; +import org.apache.hadoop.hive.conf.HiveConf; +import org.junit.Before; +import org.junit.Test; + +public class TestMetricsQueryLifeTimeHook { + + private MetricsQueryLifeTimeHook hook; + private QueryLifeTimeHookContext ctx; + private MetricRegistry metricRegistry; + + @Before + public void before() throws Exception { + HiveConf conf = new HiveConf(); + + conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, "local"); + conf.setVar(HiveConf.ConfVars.HIVE_METRICS_CLASS, CodahaleMetrics.class.getCanonicalName()); + conf.setVar(HiveConf.ConfVars.HIVE_METRICS_REPORTER, MetricsReporting.JSON_FILE.name() + + "," + MetricsReporting.JMX.name()); + conf.setVar(HiveConf.ConfVars.HIVE_METRICS_JSON_FILE_INTERVAL, "100000s"); + + MetricsFactory.init(conf); + metricRegistry = ((CodahaleMetrics) MetricsFactory.getInstance()).getMetricRegistry(); + + hook = new MetricsQueryLifeTimeHook(); + ctx = new QueryLifeTimeHookContextImpl(); + } + + @Test + public void testCompilationQueryMetric() { + Timer timer = metricRegistry.getTimers().get(MetricsConstant.HS2_COMPILING_QUERIES); + Counter counter = metricRegistry.getCounters() + .get(MetricsConstant.ACTIVE_CALLS + MetricsConstant.HS2_COMPILING_QUERIES); + assertThat(timer, nullValue()); + assertThat(counter, nullValue()); + + hook.beforeCompile(ctx); + timer = metricRegistry.getTimers().get(MetricsConstant.HS2_COMPILING_QUERIES); + counter = metricRegistry.getCounters() + .get(MetricsConstant.ACTIVE_CALLS + MetricsConstant.HS2_COMPILING_QUERIES); + assertThat(timer.getCount(), equalTo(0l)); + assertThat(counter.getCount(), equalTo(1l)); + + hook.afterCompile(ctx, false); + timer = metricRegistry.getTimers().get(MetricsConstant.HS2_COMPILING_QUERIES); + counter = metricRegistry.getCounters() + .get(MetricsConstant.ACTIVE_CALLS + MetricsConstant.HS2_COMPILING_QUERIES); + assertThat(timer.getCount(), equalTo(1l)); + assertThat(counter.getCount(), equalTo(0l)); + } + + @Test + public void testExecutionQueryMetric() { + Timer timer = metricRegistry.getTimers().get(MetricsConstant.HS2_EXECUTING_QUERIES); + Counter counter = metricRegistry.getCounters() + .get(MetricsConstant.ACTIVE_CALLS + MetricsConstant.HS2_EXECUTING_QUERIES); + assertThat(timer, nullValue()); + assertThat(counter, nullValue()); + + hook.beforeExecution(ctx); + timer = metricRegistry.getTimers().get(MetricsConstant.HS2_EXECUTING_QUERIES); + counter = metricRegistry.getCounters() + .get(MetricsConstant.ACTIVE_CALLS + MetricsConstant.HS2_EXECUTING_QUERIES); + assertThat(timer.getCount(), equalTo(0l)); + assertThat(counter.getCount(), equalTo(1l)); + + hook.afterExecution(ctx, false); + timer = metricRegistry.getTimers().get(MetricsConstant.HS2_EXECUTING_QUERIES); + counter = metricRegistry.getCounters() + .get(MetricsConstant.ACTIVE_CALLS + MetricsConstant.HS2_EXECUTING_QUERIES); + assertThat(timer.getCount(), equalTo(1l)); + assertThat(counter.getCount(), equalTo(0l)); + } + + @Test + public void testNoErrorOnDisabledMetrics() throws Exception { + MetricsFactory.close(); + MetricsQueryLifeTimeHook emptyhook = new MetricsQueryLifeTimeHook(); + + assertThat(MetricsFactory.getInstance(), nullValue()); + + emptyhook.beforeCompile(ctx); + emptyhook.afterCompile(ctx, false); + emptyhook.beforeExecution(ctx); + emptyhook.afterExecution(ctx, false); + } +} diff --git a/service/src/java/org/apache/hive/service/cli/operation/Operation.java b/service/src/java/org/apache/hive/service/cli/operation/Operation.java index 28ca41a6e992bdc0ea0db58f487b4b2a0f57f5f2..2039946b6e64c5c729c05cfedb6af298b3fa4a71 100644 --- a/service/src/java/org/apache/hive/service/cli/operation/Operation.java +++ b/service/src/java/org/apache/hive/service/cli/operation/Operation.java @@ -19,7 +19,6 @@ import java.io.File; import java.io.FileNotFoundException; -import java.io.IOException; import java.util.EnumSet; import java.util.HashMap; import java.util.List; @@ -104,7 +103,9 @@ protected Operation(HiveSession parentSession, lastAccessTime = beginTime; operationTimeout = HiveConf.getTimeVar(parentSession.getHiveConf(), HiveConf.ConfVars.HIVE_SERVER2_IDLE_OPERATION_TIMEOUT, TimeUnit.MILLISECONDS); - setMetrics(state); + + currentStateScope = updateOperationStateMetrics(null, MetricsConstant.OPERATION_PREFIX, + MetricsConstant.COMPLETED_OPERATION_PREFIX, state); queryState = new QueryState(parentSession.getHiveConf(), confOverlay, isAsyncQueryState); } @@ -163,7 +164,8 @@ protected final OperationState setState(OperationState newState) throws HiveSQLE state.validateTransition(newState); OperationState prevState = state; this.state = newState; - setMetrics(state); + currentStateScope = updateOperationStateMetrics(currentStateScope, MetricsConstant.OPERATION_PREFIX, + MetricsConstant.COMPLETED_OPERATION_PREFIX, state); onNewState(state, prevState); this.lastAccessTime = System.currentTimeMillis(); return this.state; @@ -325,11 +327,7 @@ public void run() throws HiveSQLException { try { Metrics metrics = MetricsFactory.getInstance(); if (metrics != null) { - try { - metrics.incrementCounter(MetricsConstant.OPEN_OPERATIONS); - } catch (Exception e) { - LOG.warn("Error Reporting open operation to Metrics system", e); - } + metrics.incrementCounter(MetricsConstant.OPEN_OPERATIONS); } runInternal(); } finally { @@ -414,12 +412,7 @@ protected HiveSQLException toSQLException(String prefix, CommandProcessorRespons OperationState.UNKNOWN ); - private void setMetrics(OperationState state) { - currentStateScope = setMetrics(currentStateScope, MetricsConstant.OPERATION_PREFIX, - MetricsConstant.COMPLETED_OPERATION_PREFIX, state); - } - - protected static MetricsScope setMetrics(MetricsScope stateScope, String operationPrefix, + protected final MetricsScope updateOperationStateMetrics(MetricsScope stateScope, String operationPrefix, String completedOperationPrefix, OperationState state) { Metrics metrics = MetricsFactory.getInstance(); if (metrics != null) { @@ -428,7 +421,7 @@ protected static MetricsScope setMetrics(MetricsScope stateScope, String operati stateScope = null; } if (scopeStates.contains(state)) { - stateScope = metrics.createScope(operationPrefix + state); + stateScope = metrics.createScope(MetricsConstant.API_PREFIX + operationPrefix + state); } if (terminalStates.contains(state)) { metrics.incrementCounter(completedOperationPrefix + state); diff --git a/service/src/java/org/apache/hive/service/cli/operation/SQLOperation.java b/service/src/java/org/apache/hive/service/cli/operation/SQLOperation.java index a6527560ef0b4ee071b7d8e9167d687fc7dfe8ee..668b4b718e9b636d0c34bfd549be3d82d13bed0f 100644 --- a/service/src/java/org/apache/hive/service/cli/operation/SQLOperation.java +++ b/service/src/java/org/apache/hive/service/cli/operation/SQLOperation.java @@ -107,6 +107,7 @@ */ private static Map userQueries = new HashMap(); private static final String ACTIVE_SQL_USER = MetricsConstant.SQL_OPERATION_PREFIX + "active_user"; + private MetricsScope submittedQryScp; public SQLOperation(HiveSession parentSession, String statement, Map confOverlay, boolean runInBackground, long queryTimeout) { @@ -126,6 +127,11 @@ public SQLOperation(HiveSession parentSession, String statement, MapnewHashMap(), false, 0L); + + metrics = (CodahaleMetrics) MetricsFactory.getInstance(); + } + + @After + public void tearDown() throws Exception { + MetricsFactory.getInstance().close(); + } + + @Test + public void testSubmittedQueryCount() throws Exception { + String json = ((CodahaleMetrics) metrics).dumpJson(); + MetricsTestUtils.verifyMetricsJson(json, MetricsTestUtils.TIMER, + MetricsConstant.HS2_SUBMITTED_QURIES, "0"); + + operation.onNewState(OperationState.FINISHED, OperationState.RUNNING); + + json = ((CodahaleMetrics) metrics).dumpJson(); + MetricsTestUtils.verifyMetricsJson(json, MetricsTestUtils.TIMER, + MetricsConstant.HS2_SUBMITTED_QURIES, "1"); + } + + @Test + public void testActiveUserQueriesCount() throws Exception { + String name = MetricsConstant.SQL_OPERATION_PREFIX + "active_user"; + String json = ((CodahaleMetrics) metrics).dumpJson(); + + MetricsTestUtils.verifyMetricsJson(json, MetricsTestUtils.COUNTER, name, ""); + + operation.onNewState(OperationState.RUNNING, OperationState.INITIALIZED); + json = ((CodahaleMetrics) metrics).dumpJson(); + MetricsTestUtils.verifyMetricsJson(json, MetricsTestUtils.COUNTER, name, "1"); + + operation.onNewState(OperationState.RUNNING, OperationState.RUNNING); + json = ((CodahaleMetrics) metrics).dumpJson(); + MetricsTestUtils.verifyMetricsJson(json, MetricsTestUtils.COUNTER, name, "1"); + + operation.onNewState(OperationState.FINISHED, OperationState.RUNNING); + json = ((CodahaleMetrics) metrics).dumpJson(); + MetricsTestUtils.verifyMetricsJson(json, MetricsTestUtils.COUNTER, name, "0"); + } + + @Test + public void testSucceededQueriesCount() throws Exception { + String json = ((CodahaleMetrics) metrics).dumpJson(); + + MetricsTestUtils.verifyMetricsJson(json, MetricsTestUtils.METER, + MetricsConstant.HS2_SUCEEDED_QUERIES, ""); + + operation.onNewState(OperationState.FINISHED, OperationState.RUNNING); + json = ((CodahaleMetrics) metrics).dumpJson(); + MetricsTestUtils.verifyMetricsJson(json, MetricsTestUtils.METER, + MetricsConstant.HS2_SUCEEDED_QUERIES, "1"); + + operation.onNewState(OperationState.ERROR, OperationState.RUNNING); + json = ((CodahaleMetrics) metrics).dumpJson(); + MetricsTestUtils.verifyMetricsJson(json, MetricsTestUtils.METER, + MetricsConstant.HS2_SUCEEDED_QUERIES, "1"); + + operation.onNewState(OperationState.CANCELED, OperationState.RUNNING); + json = ((CodahaleMetrics) metrics).dumpJson(); + MetricsTestUtils.verifyMetricsJson(json, MetricsTestUtils.METER, + MetricsConstant.HS2_SUCEEDED_QUERIES, "1"); + + operation.onNewState(OperationState.FINISHED, OperationState.RUNNING); + json = ((CodahaleMetrics) metrics).dumpJson(); + MetricsTestUtils.verifyMetricsJson(json, MetricsTestUtils.METER, + MetricsConstant.HS2_SUCEEDED_QUERIES, "2"); + } + + @Test + public void testFailedQueriesCount() throws Exception { + String json = ((CodahaleMetrics) metrics).dumpJson(); + + MetricsTestUtils.verifyMetricsJson(json, MetricsTestUtils.METER, + MetricsConstant.HS2_FAILED_QUERIES, ""); + + operation.onNewState(OperationState.ERROR, OperationState.RUNNING); + json = ((CodahaleMetrics) metrics).dumpJson(); + MetricsTestUtils.verifyMetricsJson(json, MetricsTestUtils.METER, + MetricsConstant.HS2_FAILED_QUERIES, "1"); + + operation.onNewState(OperationState.FINISHED, OperationState.RUNNING); + json = ((CodahaleMetrics) metrics).dumpJson(); + MetricsTestUtils.verifyMetricsJson(json, MetricsTestUtils.METER, + MetricsConstant.HS2_FAILED_QUERIES, "1"); + + operation.onNewState(OperationState.CANCELED, OperationState.RUNNING); + json = ((CodahaleMetrics) metrics).dumpJson(); + MetricsTestUtils.verifyMetricsJson(json, MetricsTestUtils.METER, + MetricsConstant.HS2_FAILED_QUERIES, "1"); + + operation.onNewState(OperationState.ERROR, OperationState.RUNNING); + json = ((CodahaleMetrics) metrics).dumpJson(); + MetricsTestUtils.verifyMetricsJson(json, MetricsTestUtils.METER, + MetricsConstant.HS2_FAILED_QUERIES, "2"); + } + + +}