diff --git common/src/java/org/apache/hadoop/hive/common/metrics/LegacyMetrics.java common/src/java/org/apache/hadoop/hive/common/metrics/LegacyMetrics.java index ba2267b..48cd9ae 100644 --- common/src/java/org/apache/hadoop/hive/common/metrics/LegacyMetrics.java +++ common/src/java/org/apache/hadoop/hive/common/metrics/LegacyMetrics.java @@ -225,6 +225,11 @@ public void addGauge(String name, MetricsVariable variable) { //Not implemented. } + @Override + public void markMeter(String name) { + //Not implemented. + } + public void set(String name, Object value) { metrics.put(name,value); } diff --git common/src/java/org/apache/hadoop/hive/common/metrics/common/Metrics.java common/src/java/org/apache/hadoop/hive/common/metrics/common/Metrics.java index 9b263d9..7eadb3b 100644 --- common/src/java/org/apache/hadoop/hive/common/metrics/common/Metrics.java +++ common/src/java/org/apache/hadoop/hive/common/metrics/common/Metrics.java @@ -17,11 +17,16 @@ */ package org.apache.hadoop.hive.common.metrics.common; +import com.codahale.metrics.Meter; + /** * Generic Metics interface. */ public interface Metrics { + public static final String API_PREFIX = "api_"; + public static final String ACTIVE_CALLS = "active_calls_"; + /** * Deinitializes the Metrics system. */ @@ -93,4 +98,11 @@ * @param variable variable to track. */ public void addGauge(String name, final MetricsVariable variable); + + /** + * Mark an event occurance for a meter. Meters measure the rate of an event and track + * 1/5/15 minute moving averages + * @param name name of the meter + */ + public void markMeter(String name); } diff --git common/src/java/org/apache/hadoop/hive/common/metrics/common/MetricsConstant.java common/src/java/org/apache/hadoop/hive/common/metrics/common/MetricsConstant.java index c9d4087..22ce6e2 100644 --- common/src/java/org/apache/hadoop/hive/common/metrics/common/MetricsConstant.java +++ common/src/java/org/apache/hadoop/hive/common/metrics/common/MetricsConstant.java @@ -68,4 +68,9 @@ // The number of tez tasks executed by the HiveServer2 since the last restart public static final String HIVE_TEZ_TASKS = "hive_tez_tasks"; + public static final String HS2_SUBMITTED_QURIES = "hs2_submitted_queries"; + public static final String HS2_COMPILING_QUERIES = "hs2_compiling_queries"; + public static final String HS2_EXECUTING_QUERIES = "hs2_executing_queries"; + public static final String HS2_FAILED_QUERIES = "hs2_failed_queries"; + public static final String HS2_SUCEEDED_QUERIES = "hs2_suceeded_queries"; } \ No newline at end of file diff --git common/src/java/org/apache/hadoop/hive/common/metrics/metrics2/CodahaleMetrics.java common/src/java/org/apache/hadoop/hive/common/metrics/metrics2/CodahaleMetrics.java index 9525b45..0f57d54 100644 --- common/src/java/org/apache/hadoop/hive/common/metrics/metrics2/CodahaleMetrics.java +++ common/src/java/org/apache/hadoop/hive/common/metrics/metrics2/CodahaleMetrics.java @@ -23,6 +23,7 @@ import com.codahale.metrics.ExponentiallyDecayingReservoir; import com.codahale.metrics.Gauge; import com.codahale.metrics.JmxReporter; +import com.codahale.metrics.Meter; import com.codahale.metrics.Metric; import com.codahale.metrics.MetricRegistry; import com.codahale.metrics.MetricSet; @@ -75,17 +76,17 @@ */ public class CodahaleMetrics implements org.apache.hadoop.hive.common.metrics.common.Metrics { - public static final String API_PREFIX = "api_"; - public static final String ACTIVE_CALLS = "active_calls_"; public static final Logger LOGGER = LoggerFactory.getLogger(CodahaleMetrics.class); public final MetricRegistry metricRegistry = new MetricRegistry(); private final Lock timersLock = new ReentrantLock(); private final Lock countersLock = new ReentrantLock(); private final Lock gaugesLock = new ReentrantLock(); + private final Lock metersLock = new ReentrantLock(); private LoadingCache timers; private LoadingCache counters; + private LoadingCache meters; private ConcurrentHashMap gauges; private HiveConf conf; @@ -168,6 +169,16 @@ public Counter load(String key) { } } ); + meters = CacheBuilder.newBuilder().build( + new CacheLoader() { + @Override + public Meter load(String key) { + Meter meter = new Meter(); + metricRegistry.register(key, meter); + return meter; + } + } + ); gauges = new ConcurrentHashMap(); //register JVM metrics @@ -208,11 +219,11 @@ public void close() throws Exception { } timers.invalidateAll(); counters.invalidateAll(); + meters.invalidateAll(); } @Override public void startStoredScope(String name) { - name = API_PREFIX + name; if (threadLocalScopes.get().containsKey(name)) { threadLocalScopes.get().get(name).open(); } else { @@ -222,7 +233,6 @@ public void startStoredScope(String name) { @Override public void endStoredScope(String name) { - name = API_PREFIX + name; if (threadLocalScopes.get().containsKey(name)) { threadLocalScopes.get().get(name).close(); threadLocalScopes.get().remove(name); @@ -238,7 +248,6 @@ public MetricsScope getStoredScope(String name) throws IllegalArgumentException } public MetricsScope createScope(String name) { - name = API_PREFIX + name; return new CodahaleMetricsScope(name); } @@ -307,6 +316,21 @@ public Object getValue() { } } + @Override + public void markMeter(String name) { + String key = name; + try { + metersLock.lock(); + Meter meter = meters.get(name); + meter.mark(); + } catch (ExecutionException e) { + throw new IllegalStateException("Error retrieving meter " + name + + " from the metric registry ", e); + } finally { + metersLock.unlock(); + } + } + // This method is necessary to synchronize lazy-creation to the timers. private Timer getTimer(String name) { String key = name; @@ -315,7 +339,8 @@ private Timer getTimer(String name) { Timer timer = timers.get(key); return timer; } catch (ExecutionException e) { - throw new IllegalStateException("Error retrieving timer from the metric registry ", e); + throw new IllegalStateException("Error retrieving timer " + name + + " from the metric registry ", e); } finally { timersLock.unlock(); } diff --git common/src/java/org/apache/hadoop/hive/ql/log/PerfLogger.java common/src/java/org/apache/hadoop/hive/ql/log/PerfLogger.java index 7658f1c..8cbbb9d 100644 --- common/src/java/org/apache/hadoop/hive/ql/log/PerfLogger.java +++ common/src/java/org/apache/hadoop/hive/ql/log/PerfLogger.java @@ -225,7 +225,7 @@ public Long getDuration(String method) { private void beginMetrics(String method) { Metrics metrics = MetricsFactory.getInstance(); if (metrics != null) { - MetricsScope scope = metrics.createScope(method); + MetricsScope scope = metrics.createScope(Metrics.API_PREFIX + method); openScopes.put(method, scope); } diff --git common/src/test/org/apache/hadoop/hive/common/metrics/MetricsTestUtils.java common/src/test/org/apache/hadoop/hive/common/metrics/MetricsTestUtils.java index 4667658..f00252c 100644 --- common/src/test/org/apache/hadoop/hive/common/metrics/MetricsTestUtils.java +++ common/src/test/org/apache/hadoop/hive/common/metrics/MetricsTestUtils.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hive.common.metrics; +import com.codahale.metrics.Meter; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import org.junit.Assert; @@ -34,6 +35,7 @@ public static final MetricsCategory COUNTER = new MetricsCategory("counters", "count"); public static final MetricsCategory TIMER = new MetricsCategory("timers", "count"); public static final MetricsCategory GAUGE = new MetricsCategory("gauges", "value"); + public static final MetricsCategory METER = new MetricsCategory("meters", "count"); static class MetricsCategory { String category; diff --git common/src/test/org/apache/hadoop/hive/common/metrics/metrics2/TestCodahaleMetrics.java common/src/test/org/apache/hadoop/hive/common/metrics/metrics2/TestCodahaleMetrics.java index 6ee6245..aa4e75f 100644 --- common/src/test/org/apache/hadoop/hive/common/metrics/metrics2/TestCodahaleMetrics.java +++ common/src/test/org/apache/hadoop/hive/common/metrics/metrics2/TestCodahaleMetrics.java @@ -80,7 +80,7 @@ public void testScope() throws Exception { MetricsFactory.getInstance().endStoredScope("method1"); } - Timer timer = metricRegistry.getTimers().get("api_method1"); + Timer timer = metricRegistry.getTimers().get("method1"); Assert.assertEquals(5, timer.getCount()); Assert.assertTrue(timer.getMeanRate() > 0); } @@ -113,7 +113,7 @@ public Void call() throws Exception { } executorService.shutdown(); assertTrue(executorService.awaitTermination(10000, TimeUnit.MILLISECONDS)); - Timer timer = metricRegistry.getTimers().get("api_method2"); + Timer timer = metricRegistry.getTimers().get("method2"); Assert.assertEquals(4, timer.getCount()); Assert.assertTrue(timer.getMeanRate() > 0); } @@ -161,4 +161,20 @@ public void testGauge() throws Exception { json = ((CodahaleMetrics) MetricsFactory.getInstance()).dumpJson(); MetricsTestUtils.verifyMetricsJson(json, MetricsTestUtils.GAUGE, "gauge1", testVar.getValue()); } + + @Test + public void testMeter() throws Exception { + + String json = ((CodahaleMetrics) MetricsFactory.getInstance()).dumpJson(); + MetricsTestUtils.verifyMetricsJson(json, MetricsTestUtils.METER, "meter", ""); + + MetricsFactory.getInstance().markMeter("meter"); + json = ((CodahaleMetrics) MetricsFactory.getInstance()).dumpJson(); + MetricsTestUtils.verifyMetricsJson(json, MetricsTestUtils.METER, "meter", "1"); + + MetricsFactory.getInstance().markMeter("meter"); + json = ((CodahaleMetrics) MetricsFactory.getInstance()).dumpJson(); + MetricsTestUtils.verifyMetricsJson(json, MetricsTestUtils.METER, "meter", "2"); + + } } diff --git metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java index 530d2f4..092c93b 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java +++ metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java @@ -762,7 +762,7 @@ private String startFunction(String function, String extraLogInfo) { logInfo((getThreadLocalIpAddress() == null ? "" : "source:" + getThreadLocalIpAddress() + " ") + function + extraLogInfo); if (MetricsFactory.getInstance() != null) { - MetricsFactory.getInstance().startStoredScope(function); + MetricsFactory.getInstance().startStoredScope(Metrics.API_PREFIX + function); } return function; } @@ -801,7 +801,7 @@ private void endFunction(String function, boolean successful, Exception e, private void endFunction(String function, MetaStoreEndFunctionContext context) { if (MetricsFactory.getInstance() != null) { - MetricsFactory.getInstance().endStoredScope(function); + MetricsFactory.getInstance().endStoredScope(Metrics.API_PREFIX + function); } for (MetaStoreEndFunctionListener listener : endFunctionListeners) { diff --git ql/src/java/org/apache/hadoop/hive/ql/Driver.java ql/src/java/org/apache/hadoop/hive/ql/Driver.java index dd55434..8fa65bb 100644 --- ql/src/java/org/apache/hadoop/hive/ql/Driver.java +++ ql/src/java/org/apache/hadoop/hive/ql/Driver.java @@ -36,6 +36,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.ReentrantLock; +import com.google.common.collect.Iterables; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.hive.common.ValidTxnList; @@ -64,6 +65,7 @@ import org.apache.hadoop.hive.ql.hooks.Hook; import org.apache.hadoop.hive.ql.hooks.HookContext; import org.apache.hadoop.hive.ql.hooks.HookUtils; +import org.apache.hadoop.hive.ql.hooks.MetricsQueryLifeTimeHook; import org.apache.hadoop.hive.ql.hooks.PostExecute; import org.apache.hadoop.hive.ql.hooks.PreExecute; import org.apache.hadoop.hive.ql.hooks.QueryLifeTimeHook; @@ -87,7 +89,6 @@ import org.apache.hadoop.hive.ql.parse.ASTNode; import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer; import org.apache.hadoop.hive.ql.parse.ColumnAccessInfo; -import org.apache.hadoop.hive.ql.parse.ExplainSemanticAnalyzer; import org.apache.hadoop.hive.ql.parse.HiveSemanticAnalyzerHook; import org.apache.hadoop.hive.ql.parse.HiveSemanticAnalyzerHookContext; import org.apache.hadoop.hive.ql.parse.HiveSemanticAnalyzerHookContextImpl; @@ -178,7 +179,7 @@ private QueryState queryState; // Query hooks that execute before compilation and after execution - List queryHooks; + private List queryHooks; private boolean checkConcurrency() { boolean supportConcurrency = conf.getBoolVar(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY); @@ -390,8 +391,8 @@ public int compile(String command, boolean resetTaskIds) { // Whether any error occurred during query compilation. Used for query lifetime hook. boolean compileError = false; - try { + // Initialize the transaction manager. This must be done before analyze is called. final HiveTxnManager txnManager = SessionState.get().initTxnMgr(conf); // In case when user Ctrl-C twice to kill Hive CLI JVM, we want to release locks @@ -425,7 +426,7 @@ public void run() { perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.PARSE); // Trigger query hook before compilation - queryHooks = getHooks(ConfVars.HIVE_QUERY_LIFETIME_HOOKS, QueryLifeTimeHook.class); + queryHooks = loadQueryHooks(); if (queryHooks != null && !queryHooks.isEmpty()) { QueryLifeTimeHookContext qhc = new QueryLifeTimeHookContextImpl(); qhc.setHiveConf(conf); @@ -572,6 +573,19 @@ public void run() { } } + private List loadQueryHooks() throws Exception { + List hooks = new ArrayList<>(); + + if (conf.getBoolVar(ConfVars.HIVE_SERVER2_METRICS_ENABLED)) { + hooks.add(new MetricsQueryLifeTimeHook()); + } + List propertyDefinedHoooks = getHooks(ConfVars.HIVE_QUERY_LIFETIME_HOOKS, QueryLifeTimeHook.class); + if (propertyDefinedHoooks != null) { + Iterables.addAll(hooks, propertyDefinedHoooks); + } + return hooks; + } + private ImmutableMap dumpMetaCallTimingWithoutEx(String phase) { try { return Hive.get().dumpAndClearMetaCallTiming(phase); @@ -1560,6 +1574,7 @@ public int execute() throws CommandNeedRetryException { perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.DRIVER_EXECUTE); boolean noName = StringUtils.isEmpty(conf.get(MRJobConfig.JOB_NAME)); int maxlen = conf.getIntVar(HiveConf.ConfVars.HIVEJOBNAMELENGTH); + Metrics metrics = MetricsFactory.getInstance(); String queryId = plan.getQueryId(); // Get the query string from the conf file as the compileInternal() method might @@ -1663,7 +1678,6 @@ public int execute() throws CommandNeedRetryException { assert tsk.getParentTasks() == null || tsk.getParentTasks().isEmpty(); driverCxt.addToRunnable(tsk); - Metrics metrics = MetricsFactory.getInstance(); if (metrics != null) { tsk.updateTaskMetrics(metrics); } @@ -2160,7 +2174,7 @@ public void setOperationId(String opId) { this.operationId = opId; } - /** + /** * Resets QueryState to get new queryId on Driver reuse. */ public void resetQueryState() { diff --git ql/src/java/org/apache/hadoop/hive/ql/hooks/MetricsQueryLifeTimeHook.java ql/src/java/org/apache/hadoop/hive/ql/hooks/MetricsQueryLifeTimeHook.java new file mode 100644 index 0000000..246ce0e --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/hooks/MetricsQueryLifeTimeHook.java @@ -0,0 +1,62 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.hooks; + +import org.apache.hadoop.hive.common.metrics.common.Metrics; +import org.apache.hadoop.hive.common.metrics.common.MetricsConstant; +import org.apache.hadoop.hive.common.metrics.common.MetricsFactory; +import org.apache.hadoop.hive.common.metrics.common.MetricsScope; + +/** + * LifeTimeHook gathering metrics for the query lifecycle if the + * metrics are enabled + */ +public class MetricsQueryLifeTimeHook implements QueryLifeTimeHook { + + private Metrics metrics = MetricsFactory.getInstance(); + private MetricsScope compilingQryScp; + private MetricsScope executingQryScp; + + @Override + public void beforeCompile(QueryLifeTimeHookContext ctx) { + if (metrics != null) { + compilingQryScp = metrics.createScope(MetricsConstant.HS2_COMPILING_QUERIES); + } + } + + @Override + public void afterCompile(QueryLifeTimeHookContext ctx, boolean hasError) { + if (metrics != null && compilingQryScp != null) { + metrics.endScope(compilingQryScp); + } + } + + @Override + public void beforeExecution(QueryLifeTimeHookContext ctx) { + if (metrics != null) { + executingQryScp = metrics.createScope(MetricsConstant.HS2_EXECUTING_QUERIES); + } + } + + @Override + public void afterExecution(QueryLifeTimeHookContext ctx, boolean hasError) { + if (metrics != null && executingQryScp != null) { + metrics.endScope(executingQryScp); + } + } +} diff --git ql/src/test/org/apache/hadoop/hive/ql/hooks/TestMetricsQueryLifeTimeHook.java ql/src/test/org/apache/hadoop/hive/ql/hooks/TestMetricsQueryLifeTimeHook.java new file mode 100644 index 0000000..7a584f1 --- /dev/null +++ ql/src/test/org/apache/hadoop/hive/ql/hooks/TestMetricsQueryLifeTimeHook.java @@ -0,0 +1,98 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.hooks; + +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.nullValue; +import static org.junit.Assert.assertThat; + +import com.codahale.metrics.MetricRegistry; +import com.codahale.metrics.Timer; +import org.apache.hadoop.fs.CommonConfigurationKeysPublic; +import org.apache.hadoop.hive.common.metrics.common.MetricsConstant; +import org.apache.hadoop.hive.common.metrics.common.MetricsFactory; +import org.apache.hadoop.hive.common.metrics.metrics2.CodahaleMetrics; +import org.apache.hadoop.hive.common.metrics.metrics2.MetricsReporting; +import org.apache.hadoop.hive.conf.HiveConf; +import org.junit.Before; +import org.junit.Test; + +public class TestMetricsQueryLifeTimeHook { + + private MetricsQueryLifeTimeHook hook; + private QueryLifeTimeHookContext ctx; + private MetricRegistry metricRegistry; + + @Before + public void before() throws Exception { + HiveConf conf = new HiveConf(); + + conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, "local"); + conf.setVar(HiveConf.ConfVars.HIVE_METRICS_CLASS, CodahaleMetrics.class.getCanonicalName()); + conf.setVar(HiveConf.ConfVars.HIVE_METRICS_REPORTER, MetricsReporting.JSON_FILE.name() + + "," + MetricsReporting.JMX.name()); + conf.setVar(HiveConf.ConfVars.HIVE_METRICS_JSON_FILE_INTERVAL, "100000s"); + + MetricsFactory.init(conf); + metricRegistry = ((CodahaleMetrics) MetricsFactory.getInstance()).getMetricRegistry(); + + hook = new MetricsQueryLifeTimeHook(); + ctx = new QueryLifeTimeHookContextImpl(); + } + + @Test + public void testCompilationQueryMetric() { + Timer timer = metricRegistry.getTimers().get(MetricsConstant.HS2_COMPILING_QUERIES); + assertThat(timer, nullValue()); + + hook.beforeCompile(ctx); + timer = metricRegistry.getTimers().get(MetricsConstant.HS2_COMPILING_QUERIES); + assertThat(timer.getCount(), equalTo(0l)); + + hook.afterCompile(ctx, false); + timer = metricRegistry.getTimers().get(MetricsConstant.HS2_COMPILING_QUERIES); + assertThat(timer.getCount(), equalTo(1l)); + } + + @Test + public void testExecutionQueryMetric() { + Timer timer = metricRegistry.getTimers().get(MetricsConstant.HS2_EXECUTING_QUERIES); + assertThat(timer, nullValue()); + + hook.beforeExecution(ctx); + timer = metricRegistry.getTimers().get(MetricsConstant.HS2_EXECUTING_QUERIES); + assertThat(timer.getCount(), equalTo(0l)); + + hook.afterExecution(ctx, false); + timer = metricRegistry.getTimers().get(MetricsConstant.HS2_EXECUTING_QUERIES); + assertThat(timer.getCount(), equalTo(1l)); + } + + @Test + public void testNoErrorOnDisabledMetrics() throws Exception { + MetricsFactory.close(); + MetricsQueryLifeTimeHook emptyhook = new MetricsQueryLifeTimeHook(); + + assertThat(MetricsFactory.getInstance(), nullValue()); + + emptyhook.beforeCompile(ctx); + emptyhook.afterCompile(ctx, false); + emptyhook.beforeExecution(ctx); + emptyhook.afterExecution(ctx, false); + } +} diff --git service/src/java/org/apache/hive/service/cli/operation/Operation.java service/src/java/org/apache/hive/service/cli/operation/Operation.java index 36c6f93..dca49e2 100644 --- service/src/java/org/apache/hive/service/cli/operation/Operation.java +++ service/src/java/org/apache/hive/service/cli/operation/Operation.java @@ -19,7 +19,6 @@ import java.io.File; import java.io.FileNotFoundException; -import java.io.IOException; import java.util.EnumSet; import java.util.HashMap; import java.util.List; @@ -105,7 +104,9 @@ protected Operation(HiveSession parentSession, lastAccessTime = beginTime; operationTimeout = HiveConf.getTimeVar(parentSession.getHiveConf(), HiveConf.ConfVars.HIVE_SERVER2_IDLE_OPERATION_TIMEOUT, TimeUnit.MILLISECONDS); - setMetrics(state); + + currentStateScope = updateOperationStateMetrics(null, MetricsConstant.OPERATION_PREFIX, + MetricsConstant.COMPLETED_OPERATION_PREFIX, state); queryState = new QueryState(parentSession.getHiveConf(), confOverlay, isAsyncQueryState); } @@ -164,7 +165,8 @@ protected final OperationState setState(OperationState newState) throws HiveSQLE state.validateTransition(newState); OperationState prevState = state; this.state = newState; - setMetrics(state); + currentStateScope = updateOperationStateMetrics(currentStateScope, MetricsConstant.OPERATION_PREFIX, + MetricsConstant.COMPLETED_OPERATION_PREFIX, state); onNewState(state, prevState); this.lastAccessTime = System.currentTimeMillis(); return this.state; @@ -326,11 +328,7 @@ public void run() throws HiveSQLException { try { Metrics metrics = MetricsFactory.getInstance(); if (metrics != null) { - try { - metrics.incrementCounter(MetricsConstant.OPEN_OPERATIONS); - } catch (Exception e) { - LOG.warn("Error Reporting open operation to Metrics system", e); - } + metrics.incrementCounter(MetricsConstant.OPEN_OPERATIONS); } runInternal(); } finally { @@ -419,12 +417,7 @@ protected HiveSQLException toSQLException(String prefix, CommandProcessorRespons OperationState.UNKNOWN ); - private void setMetrics(OperationState state) { - currentStateScope = setMetrics(currentStateScope, MetricsConstant.OPERATION_PREFIX, - MetricsConstant.COMPLETED_OPERATION_PREFIX, state); - } - - protected static MetricsScope setMetrics(MetricsScope stateScope, String operationPrefix, + protected final MetricsScope updateOperationStateMetrics(MetricsScope stateScope, String operationPrefix, String completedOperationPrefix, OperationState state) { Metrics metrics = MetricsFactory.getInstance(); if (metrics != null) { @@ -433,7 +426,7 @@ protected static MetricsScope setMetrics(MetricsScope stateScope, String operati stateScope = null; } if (scopeStates.contains(state)) { - stateScope = metrics.createScope(operationPrefix + state); + stateScope = metrics.createScope(Metrics.API_PREFIX + operationPrefix + state); } if (terminalStates.contains(state)) { metrics.incrementCounter(completedOperationPrefix + state); diff --git service/src/java/org/apache/hive/service/cli/operation/SQLOperation.java service/src/java/org/apache/hive/service/cli/operation/SQLOperation.java index abdf8cd..7d6ffb9 100644 --- service/src/java/org/apache/hive/service/cli/operation/SQLOperation.java +++ service/src/java/org/apache/hive/service/cli/operation/SQLOperation.java @@ -107,6 +107,7 @@ */ private static Map userQueries = new HashMap(); private static final String ACTIVE_SQL_USER = MetricsConstant.SQL_OPERATION_PREFIX + "active_user"; + private MetricsScope submittedQryScp; public SQLOperation(HiveSession parentSession, String statement, Map confOverlay, boolean runInBackground, long queryTimeout) { @@ -126,6 +127,11 @@ public SQLOperation(HiveSession parentSession, String statement, MapnewHashMap(), false, 0L); + + metrics = (CodahaleMetrics) MetricsFactory.getInstance(); + } + + @After + public void tearDown() throws Exception { + MetricsFactory.getInstance().close(); + } + + @Test + public void testSubmittedQueryCount() throws Exception { + String json = ((CodahaleMetrics) metrics).dumpJson(); + MetricsTestUtils.verifyMetricsJson(json, MetricsTestUtils.TIMER, + MetricsConstant.HS2_SUBMITTED_QURIES, "0"); + + operation.onNewState(OperationState.FINISHED, OperationState.RUNNING); + + json = ((CodahaleMetrics) metrics).dumpJson(); + MetricsTestUtils.verifyMetricsJson(json, MetricsTestUtils.TIMER, + MetricsConstant.HS2_SUBMITTED_QURIES, "1"); + } + + @Test + public void testActiveUserQueriesCount() throws Exception { + String name = MetricsConstant.SQL_OPERATION_PREFIX + "active_user"; + String json = ((CodahaleMetrics) metrics).dumpJson(); + + MetricsTestUtils.verifyMetricsJson(json, MetricsTestUtils.COUNTER, name, ""); + + operation.onNewState(OperationState.RUNNING, OperationState.INITIALIZED); + json = ((CodahaleMetrics) metrics).dumpJson(); + MetricsTestUtils.verifyMetricsJson(json, MetricsTestUtils.COUNTER, name, "1"); + + operation.onNewState(OperationState.RUNNING, OperationState.RUNNING); + json = ((CodahaleMetrics) metrics).dumpJson(); + MetricsTestUtils.verifyMetricsJson(json, MetricsTestUtils.COUNTER, name, "1"); + + operation.onNewState(OperationState.FINISHED, OperationState.RUNNING); + json = ((CodahaleMetrics) metrics).dumpJson(); + MetricsTestUtils.verifyMetricsJson(json, MetricsTestUtils.COUNTER, name, "0"); + } + + @Test + public void testSucceededQueriesCount() throws Exception { + String json = ((CodahaleMetrics) metrics).dumpJson(); + + MetricsTestUtils.verifyMetricsJson(json, MetricsTestUtils.METER, + MetricsConstant.HS2_SUCEEDED_QUERIES, ""); + + operation.onNewState(OperationState.FINISHED, OperationState.RUNNING); + json = ((CodahaleMetrics) metrics).dumpJson(); + MetricsTestUtils.verifyMetricsJson(json, MetricsTestUtils.METER, + MetricsConstant.HS2_SUCEEDED_QUERIES, "1"); + + operation.onNewState(OperationState.ERROR, OperationState.RUNNING); + json = ((CodahaleMetrics) metrics).dumpJson(); + MetricsTestUtils.verifyMetricsJson(json, MetricsTestUtils.METER, + MetricsConstant.HS2_SUCEEDED_QUERIES, "1"); + + operation.onNewState(OperationState.CANCELED, OperationState.RUNNING); + json = ((CodahaleMetrics) metrics).dumpJson(); + MetricsTestUtils.verifyMetricsJson(json, MetricsTestUtils.METER, + MetricsConstant.HS2_SUCEEDED_QUERIES, "1"); + + operation.onNewState(OperationState.FINISHED, OperationState.RUNNING); + json = ((CodahaleMetrics) metrics).dumpJson(); + MetricsTestUtils.verifyMetricsJson(json, MetricsTestUtils.METER, + MetricsConstant.HS2_SUCEEDED_QUERIES, "2"); + } + + @Test + public void testFailedQueriesCount() throws Exception { + String json = ((CodahaleMetrics) metrics).dumpJson(); + + MetricsTestUtils.verifyMetricsJson(json, MetricsTestUtils.METER, + MetricsConstant.HS2_FAILED_QUERIES, ""); + + operation.onNewState(OperationState.ERROR, OperationState.RUNNING); + json = ((CodahaleMetrics) metrics).dumpJson(); + MetricsTestUtils.verifyMetricsJson(json, MetricsTestUtils.METER, + MetricsConstant.HS2_FAILED_QUERIES, "1"); + + operation.onNewState(OperationState.FINISHED, OperationState.RUNNING); + json = ((CodahaleMetrics) metrics).dumpJson(); + MetricsTestUtils.verifyMetricsJson(json, MetricsTestUtils.METER, + MetricsConstant.HS2_FAILED_QUERIES, "1"); + + operation.onNewState(OperationState.CANCELED, OperationState.RUNNING); + json = ((CodahaleMetrics) metrics).dumpJson(); + MetricsTestUtils.verifyMetricsJson(json, MetricsTestUtils.METER, + MetricsConstant.HS2_FAILED_QUERIES, "1"); + + operation.onNewState(OperationState.ERROR, OperationState.RUNNING); + json = ((CodahaleMetrics) metrics).dumpJson(); + MetricsTestUtils.verifyMetricsJson(json, MetricsTestUtils.METER, + MetricsConstant.HS2_FAILED_QUERIES, "2"); + } + + +}