diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 99c26ce..8f39b47 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -2266,7 +2266,7 @@ private static void populateLlapDaemonVarsSet(Set llapDaemonVarsSetLocal "Component name to provide to Hadoop2 Metrics system. Ideally 'hivemetastore' for the MetaStore " + " and and 'hiveserver2' for HiveServer2." ), - HIVE_PERF_LOGGER("hive.exec.perf.logger", "org.apache.hadoop.hive.ql.log.PerfLogger", + HIVE_PERF_LOGGER("hive.exec.perf.logger", "org.apache.hadoop.hive.ql.log.PerfLoggerImpl", "The class responsible for logging client side performance metrics. \n" + "Must be a subclass of org.apache.hadoop.hive.ql.log.PerfLogger"), HIVE_START_CLEANUP_SCRATCHDIR("hive.start.cleanup.scratchdir", false, diff --git a/common/src/java/org/apache/hadoop/hive/ql/log/PerfLogger.java b/common/src/java/org/apache/hadoop/hive/ql/log/PerfLogger.java index 008b3b0..a0740c3 100644 --- a/common/src/java/org/apache/hadoop/hive/ql/log/PerfLogger.java +++ b/common/src/java/org/apache/hadoop/hive/ql/log/PerfLogger.java @@ -15,131 +15,28 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.hadoop.hive.ql.log; import com.google.common.collect.ImmutableMap; -import org.apache.hadoop.hive.common.metrics.common.Metrics; -import org.apache.hadoop.hive.common.metrics.common.MetricsConstant; -import org.apache.hadoop.hive.common.metrics.common.MetricsFactory; -import org.apache.hadoop.hive.common.metrics.common.MetricsScope; -import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.util.ReflectionUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.HashMap; -import java.util.Map; /** - * PerfLogger. - * - * Can be used to measure and log the time spent by a piece of code. + * Interface for Performance Logger */ -public class PerfLogger { - public static final String ACQUIRE_READ_WRITE_LOCKS = "acquireReadWriteLocks"; - public static final String COMPILE = "compile"; - public static final String WAIT_COMPILE = "waitCompile"; - public static final String PARSE = "parse"; - public static final String ANALYZE = "semanticAnalyze"; - public static final String OPTIMIZER = "optimizer"; - public static final String DO_AUTHORIZATION = "doAuthorization"; - public static final String DRIVER_EXECUTE = "Driver.execute"; - public static final String INPUT_SUMMARY = "getInputSummary"; - public static final String GET_SPLITS = "getSplits"; - public static final String RUN_TASKS = "runTasks"; - public static final String SERIALIZE_PLAN = "serializePlan"; - public static final String DESERIALIZE_PLAN = "deserializePlan"; - public static final String CLONE_PLAN = "clonePlan"; - public static final String TASK = "task."; - public static final String RELEASE_LOCKS = "releaseLocks"; - public static final String PRUNE_LISTING = "prune-listing"; - public static final String PARTITION_RETRIEVING = "partition-retrieving"; - public static final String PRE_HOOK = "PreHook."; - public static final String POST_HOOK = "PostHook."; - public static final String FAILURE_HOOK = "FailureHook."; - public static final String DRIVER_RUN = "Driver.run"; - public static final String TEZ_COMPILER = "TezCompiler"; - public static final String TEZ_SUBMIT_TO_RUNNING = "TezSubmitToRunningDag"; - public static final String TEZ_BUILD_DAG = "TezBuildDag"; - public static final String TEZ_SUBMIT_DAG = "TezSubmitDag"; - public static final String TEZ_RUN_DAG = "TezRunDag"; - public static final String TEZ_CREATE_VERTEX = "TezCreateVertex."; - public static final String TEZ_RUN_VERTEX = "TezRunVertex."; - public static final String TEZ_INITIALIZE_PROCESSOR = "TezInitializeProcessor"; - public static final String TEZ_RUN_PROCESSOR = "TezRunProcessor"; - public static final String TEZ_INIT_OPERATORS = "TezInitializeOperators"; - public static final String LOAD_HASHTABLE = "LoadHashtable"; - - public static final String SPARK_SUBMIT_TO_RUNNING = "SparkSubmitToRunning"; - public static final String SPARK_BUILD_PLAN = "SparkBuildPlan"; - public static final String SPARK_BUILD_RDD_GRAPH = "SparkBuildRDDGraph"; - public static final String SPARK_SUBMIT_JOB = "SparkSubmitJob"; - public static final String SPARK_RUN_JOB = "SparkRunJob"; - public static final String SPARK_CREATE_TRAN = "SparkCreateTran."; - public static final String SPARK_RUN_STAGE = "SparkRunStage."; - public static final String SPARK_INIT_OPERATORS = "SparkInitializeOperators"; - public static final String SPARK_GENERATE_TASK_TREE = "SparkGenerateTaskTree"; - public static final String SPARK_OPTIMIZE_OPERATOR_TREE = "SparkOptimizeOperatorTree"; - public static final String SPARK_OPTIMIZE_TASK_TREE = "SparkOptimizeTaskTree"; - public static final String SPARK_FLUSH_HASHTABLE = "SparkFlushHashTable."; - - protected final Map startTimes = new HashMap(); - protected final Map endTimes = new HashMap(); - - static final private Logger LOG = LoggerFactory.getLogger(PerfLogger.class.getName()); - protected static final ThreadLocal perfLogger = new ThreadLocal(); - - - private PerfLogger() { - // Use getPerfLogger to get an instance of PerfLogger - } - - public static PerfLogger getPerfLogger(HiveConf conf, boolean resetPerfLogger) { - PerfLogger result = perfLogger.get(); - if (resetPerfLogger || result == null) { - if (conf == null) { - result = new PerfLogger(); - } else { - try { - result = (PerfLogger) ReflectionUtils.newInstance(conf.getClassByName( - conf.getVar(HiveConf.ConfVars.HIVE_PERF_LOGGER)), conf); - } catch (ClassNotFoundException e) { - LOG.error("Performance Logger Class not found:" + e.getMessage()); - result = new PerfLogger(); - } - } - perfLogger.set(result); - } - return result; - } - - public static void setPerfLogger(PerfLogger resetPerfLogger) { - perfLogger.set(resetPerfLogger); - } - - /** +public interface PerfLogger { + /** * Call this function when you start to measure time spent by a piece of code. * @param callerName the logging object to be used. * @param method method or ID that identifies this perf log element. */ - public void PerfLogBegin(String callerName, String method) { - long startTime = System.currentTimeMillis(); - startTimes.put(method, new Long(startTime)); - if (LOG.isDebugEnabled()) { - LOG.debug(""); - } - beginMetrics(method); - } + public void PerfLogBegin(String callerName, String method); + /** * Call this function in correspondence of PerfLogBegin to mark the end of the measurement. * @param callerName * @param method * @return long duration the difference between now and startTime, or -1 if startTime is null */ - public long PerfLogEnd(String callerName, String method) { - return PerfLogEnd(callerName, method, null); - } + public long PerfLogEnd(String callerName, String method); /** * Call this function in correspondence of PerfLogBegin to mark the end of the measurement. @@ -147,107 +44,24 @@ public long PerfLogEnd(String callerName, String method) { * @param method * @return long duration the difference between now and startTime, or -1 if startTime is null */ - public long PerfLogEnd(String callerName, String method, String additionalInfo) { - Long startTime = startTimes.get(method); - long endTime = System.currentTimeMillis(); - endTimes.put(method, new Long(endTime)); - long duration = startTime == null ? -1 : endTime - startTime.longValue(); - - if (LOG.isDebugEnabled()) { - StringBuilder sb = new StringBuilder(""); - LOG.debug(sb.toString()); - } - endMetrics(method); - return duration; - } - - public Long getStartTime(String method) { - long startTime = 0L; - - if (startTimes.containsKey(method)) { - startTime = startTimes.get(method); - } - return startTime; - } - - public Long getEndTime(String method) { - long endTime = 0L; - - if (endTimes.containsKey(method)) { - endTime = endTimes.get(method); - } - return endTime; - } - - public boolean startTimeHasMethod(String method) { - return startTimes.containsKey(method); - } - - public boolean endTimeHasMethod(String method) { - return endTimes.containsKey(method); - } - - public Long getDuration(String method) { - long duration = 0; - if (startTimes.containsKey(method) && endTimes.containsKey(method)) { - duration = endTimes.get(method) - startTimes.get(method); - } - return duration; - } + public long PerfLogEnd(String callerName, String method, String additionalInfo); + public Long getStartTime(String method); - public ImmutableMap getStartTimes() { - return ImmutableMap.copyOf(startTimes); - } + public Long getEndTime(String method); - public ImmutableMap getEndTimes() { - return ImmutableMap.copyOf(endTimes); - } + public boolean startTimeHasMethod(String method); - //Methods for metrics integration. Each thread-local PerfLogger will open/close scope during each perf-log method. - transient Map openScopes = new HashMap(); + public boolean endTimeHasMethod(String method); - private void beginMetrics(String method) { - Metrics metrics = MetricsFactory.getInstance(); - if (metrics != null) { - MetricsScope scope = metrics.createScope(MetricsConstant.API_PREFIX + method); - openScopes.put(method, scope); - } + public Long getDuration(String method); - } + public ImmutableMap getStartTimes(); - private void endMetrics(String method) { - Metrics metrics = MetricsFactory.getInstance(); - if (metrics != null) { - MetricsScope scope = openScopes.remove(method); - if (scope != null) { - metrics.endScope(scope); - } - } - } + public ImmutableMap getEndTimes(); /** * Cleans up any dangling perfLog metric call scopes. */ - public void cleanupPerfLogMetrics() { - Metrics metrics = MetricsFactory.getInstance(); - if (metrics != null) { - for (MetricsScope openScope : openScopes.values()) { - metrics.endScope(openScope); - } - } - openScopes.clear(); - } -} + public void cleanupPerfLogMetrics(); +} \ No newline at end of file diff --git a/common/src/java/org/apache/hadoop/hive/ql/log/PerfLoggerFactory.java b/common/src/java/org/apache/hadoop/hive/ql/log/PerfLoggerFactory.java new file mode 100644 index 0000000..e7d54f5 --- /dev/null +++ b/common/src/java/org/apache/hadoop/hive/ql/log/PerfLoggerFactory.java @@ -0,0 +1,62 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.log; + +import java.util.logging.Level; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.util.ReflectionUtils; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * PerfLogger factory, honors hive.exec.perf.logger + */ +public final class PerfLoggerFactory { + // We use the PerfLogger class logger, for back-compat + private static final Logger LOG = LoggerFactory.getLogger(PerfLogger.class.getName()); + + private static final ThreadLocal perfLogger = new ThreadLocal<>(); + + private PerfLoggerFactory() { + // Never instantiated + } + + public static PerfLogger getPerfLogger(HiveConf conf, boolean resetPerfLogger) { + PerfLogger result = perfLogger.get(); + if (resetPerfLogger || result == null) { + if (conf == null) { + result = PerfLoggerImpl.createForFactory(); + } else { + try { + result = (PerfLogger) ReflectionUtils.newInstance(conf.getClassByName( + conf.getVar(HiveConf.ConfVars.HIVE_PERF_LOGGER)), conf); + } catch (RuntimeException | ClassNotFoundException e) { + LOG.error("Performance Logger Class not found:" + e.getMessage()); + result = PerfLoggerImpl.createForFactory(); + } + } + perfLogger.set(result); + } + return result; + } + + public static void setPerfLogger(PerfLogger resetPerfLogger) { + perfLogger.set(resetPerfLogger); + } +} diff --git a/common/src/java/org/apache/hadoop/hive/ql/log/PerfLoggerImpl.java b/common/src/java/org/apache/hadoop/hive/ql/log/PerfLoggerImpl.java new file mode 100644 index 0000000..452d79e --- /dev/null +++ b/common/src/java/org/apache/hadoop/hive/ql/log/PerfLoggerImpl.java @@ -0,0 +1,174 @@ +/* + * To change this license header, choose License Headers in Project Properties. + * To change this template file, choose Tools | Templates + * and open the template in the editor. + */ +package org.apache.hadoop.hive.ql.log; + +import com.google.common.collect.ImmutableMap; +import java.util.HashMap; +import java.util.Map; + +import org.apache.hadoop.hive.common.metrics.common.Metrics; +import org.apache.hadoop.hive.common.metrics.common.MetricsConstant; +import org.apache.hadoop.hive.common.metrics.common.MetricsFactory; +import org.apache.hadoop.hive.common.metrics.common.MetricsScope; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Default implementation of PerfLogger + * Logs as LOG.debug into hive log + */ +public class PerfLoggerImpl implements PerfLogger { + protected final Map startTimes = new HashMap(); + protected final Map endTimes = new HashMap(); + + // Use the PerfLogger logger, for back-compat with existing scripts and conf + static final private Logger LOG = LoggerFactory.getLogger(PerfLogger.class.getName()); + + + /** + * Use PerfLoggerFactory.getPerfLogger to get the current logger + */ + private PerfLoggerImpl() { + } + + static PerfLogger createForFactory() { + return new PerfLoggerImpl(); + } + + /** + * Call this function when you start to measure time spent by a piece of code. + * @param callerName the logging object to be used. + * @param method method or ID that identifies this perf log element. + */ + public void PerfLogBegin(String callerName, String method) { + long startTime = System.currentTimeMillis(); + startTimes.put(method, new Long(startTime)); + if (LOG.isDebugEnabled()) { + LOG.debug(""); + } + beginMetrics(method); + } + /** + * Call this function in correspondence of PerfLogBegin to mark the end of the measurement. + * @param callerName + * @param method + * @return long duration the difference between now and startTime, or -1 if startTime is null + */ + public long PerfLogEnd(String callerName, String method) { + return PerfLogEnd(callerName, method, null); + } + + /** + * Call this function in correspondence of PerfLogBegin to mark the end of the measurement. + * @param callerName + * @param method + * @return long duration the difference between now and startTime, or -1 if startTime is null + */ + public long PerfLogEnd(String callerName, String method, String additionalInfo) { + Long startTime = startTimes.get(method); + long endTime = System.currentTimeMillis(); + endTimes.put(method, new Long(endTime)); + long duration = startTime == null ? -1 : endTime - startTime.longValue(); + + if (LOG.isDebugEnabled()) { + StringBuilder sb = new StringBuilder(""); + LOG.debug(sb.toString()); + } + endMetrics(method); + return duration; + } + + public Long getStartTime(String method) { + long startTime = 0L; + + if (startTimes.containsKey(method)) { + startTime = startTimes.get(method); + } + return startTime; + } + + public Long getEndTime(String method) { + long endTime = 0L; + + if (endTimes.containsKey(method)) { + endTime = endTimes.get(method); + } + return endTime; + } + + public boolean startTimeHasMethod(String method) { + return startTimes.containsKey(method); + } + + public boolean endTimeHasMethod(String method) { + return endTimes.containsKey(method); + } + + public Long getDuration(String method) { + long duration = 0; + if (startTimes.containsKey(method) && endTimes.containsKey(method)) { + duration = endTimes.get(method) - startTimes.get(method); + } + return duration; + } + + + public ImmutableMap getStartTimes() { + return ImmutableMap.copyOf(startTimes); + } + + public ImmutableMap getEndTimes() { + return ImmutableMap.copyOf(endTimes); + } + + //Methods for metrics integration. Each thread-local PerfLoggerTokens will open/close scope during each perf-log method. + transient Map openScopes = new HashMap(); + + private void beginMetrics(String method) { + Metrics metrics = MetricsFactory.getInstance(); + if (metrics != null) { + MetricsScope scope = metrics.createScope(MetricsConstant.API_PREFIX + method); + openScopes.put(method, scope); + } + + } + + private void endMetrics(String method) { + Metrics metrics = MetricsFactory.getInstance(); + if (metrics != null) { + MetricsScope scope = openScopes.remove(method); + if (scope != null) { + metrics.endScope(scope); + } + } + } + + /** + * Cleans up any dangling perfLog metric call scopes. + */ + public void cleanupPerfLogMetrics() { + Metrics metrics = MetricsFactory.getInstance(); + if (metrics != null) { + for (MetricsScope openScope : openScopes.values()) { + metrics.endScope(openScope); + } + } + openScopes.clear(); + } +} diff --git a/common/src/java/org/apache/hadoop/hive/ql/log/PerfLoggerTokens.java b/common/src/java/org/apache/hadoop/hive/ql/log/PerfLoggerTokens.java new file mode 100644 index 0000000..4435e45 --- /dev/null +++ b/common/src/java/org/apache/hadoop/hive/ql/log/PerfLoggerTokens.java @@ -0,0 +1,73 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.log; + +/** + * PerfLoggerTokens. + * + * String tokens used by perf logger clients + */ +public class PerfLoggerTokens { + public static final String ACQUIRE_READ_WRITE_LOCKS = "acquireReadWriteLocks"; + public static final String COMPILE = "compile"; + public static final String WAIT_COMPILE = "waitCompile"; + public static final String PARSE = "parse"; + public static final String ANALYZE = "semanticAnalyze"; + public static final String OPTIMIZER = "optimizer"; + public static final String DO_AUTHORIZATION = "doAuthorization"; + public static final String DRIVER_EXECUTE = "Driver.execute"; + public static final String INPUT_SUMMARY = "getInputSummary"; + public static final String GET_SPLITS = "getSplits"; + public static final String RUN_TASKS = "runTasks"; + public static final String SERIALIZE_PLAN = "serializePlan"; + public static final String DESERIALIZE_PLAN = "deserializePlan"; + public static final String CLONE_PLAN = "clonePlan"; + public static final String TASK = "task."; + public static final String RELEASE_LOCKS = "releaseLocks"; + public static final String PRUNE_LISTING = "prune-listing"; + public static final String PARTITION_RETRIEVING = "partition-retrieving"; + public static final String PRE_HOOK = "PreHook."; + public static final String POST_HOOK = "PostHook."; + public static final String FAILURE_HOOK = "FailureHook."; + public static final String DRIVER_RUN = "Driver.run"; + public static final String TEZ_COMPILER = "TezCompiler"; + public static final String TEZ_SUBMIT_TO_RUNNING = "TezSubmitToRunningDag"; + public static final String TEZ_BUILD_DAG = "TezBuildDag"; + public static final String TEZ_SUBMIT_DAG = "TezSubmitDag"; + public static final String TEZ_RUN_DAG = "TezRunDag"; + public static final String TEZ_CREATE_VERTEX = "TezCreateVertex."; + public static final String TEZ_RUN_VERTEX = "TezRunVertex."; + public static final String TEZ_INITIALIZE_PROCESSOR = "TezInitializeProcessor"; + public static final String TEZ_RUN_PROCESSOR = "TezRunProcessor"; + public static final String TEZ_INIT_OPERATORS = "TezInitializeOperators"; + public static final String LOAD_HASHTABLE = "LoadHashtable"; + + public static final String SPARK_SUBMIT_TO_RUNNING = "SparkSubmitToRunning"; + public static final String SPARK_BUILD_PLAN = "SparkBuildPlan"; + public static final String SPARK_BUILD_RDD_GRAPH = "SparkBuildRDDGraph"; + public static final String SPARK_SUBMIT_JOB = "SparkSubmitJob"; + public static final String SPARK_RUN_JOB = "SparkRunJob"; + public static final String SPARK_CREATE_TRAN = "SparkCreateTran."; + public static final String SPARK_RUN_STAGE = "SparkRunStage."; + public static final String SPARK_INIT_OPERATORS = "SparkInitializeOperators"; + public static final String SPARK_GENERATE_TASK_TREE = "SparkGenerateTaskTree"; + public static final String SPARK_OPTIMIZE_OPERATOR_TREE = "SparkOptimizeOperatorTree"; + public static final String SPARK_OPTIMIZE_TASK_TREE = "SparkOptimizeTaskTree"; + public static final String SPARK_FLUSH_HASHTABLE = "SparkFlushHashTable."; +} diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/RetryingHMSHandler.java b/metastore/src/java/org/apache/hadoop/hive/metastore/RetryingHMSHandler.java index b1c8e39..3e9c03e 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/RetryingHMSHandler.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/RetryingHMSHandler.java @@ -35,6 +35,8 @@ import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; import org.apache.hadoop.hive.ql.log.PerfLogger; +import org.apache.hadoop.hive.ql.log.PerfLoggerFactory; +import org.apache.hadoop.hive.ql.log.PerfLoggerTokens; import org.datanucleus.exceptions.NucleusException; @InterfaceAudience.Private @@ -101,7 +103,7 @@ public Object invoke(final Object proxy, final Method method, final Object[] arg int retryCount = -1; int threadId = HiveMetaStore.HMSHandler.get(); boolean error = true; - PerfLogger perfLogger = PerfLogger.getPerfLogger(origConf, false); + PerfLogger perfLogger = PerfLoggerFactory.getPerfLogger(origConf, false); perfLogger.PerfLogBegin(CLASS_NAME, method.getName()); try { Result result = invokeInternal(proxy, method, args); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java index 29cce9a..34dd6a2 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java @@ -78,6 +78,7 @@ import org.apache.hadoop.hive.ql.lockmgr.HiveTxnManager; import org.apache.hadoop.hive.ql.lockmgr.LockException; import org.apache.hadoop.hive.ql.log.PerfLogger; +import org.apache.hadoop.hive.ql.log.PerfLoggerTokens; import org.apache.hadoop.hive.ql.metadata.AuthorizationException; import org.apache.hadoop.hive.ql.metadata.Hive; import org.apache.hadoop.hive.ql.metadata.HiveException; @@ -414,8 +415,8 @@ public int compile(String command, boolean resetTaskIds) { // runInternal, which defers the close to the called in that method. private int compile(String command, boolean resetTaskIds, boolean deferClose) { PerfLogger perfLogger = SessionState.getPerfLogger(true); - perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.DRIVER_RUN); - perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.COMPILE); + perfLogger.PerfLogBegin(CLASS_NAME, PerfLoggerTokens.DRIVER_RUN); + perfLogger.PerfLogBegin(CLASS_NAME, PerfLoggerTokens.COMPILE); lDrvState.stateLock.lock(); try { lDrvState.driverState = DriverState.COMPILING; @@ -500,7 +501,7 @@ public void run() { ctx.setCmd(command); ctx.setHDFSCleanup(true); - perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.PARSE); + perfLogger.PerfLogBegin(CLASS_NAME, PerfLoggerTokens.PARSE); // Trigger query hook before compilation queryLifeTimeHookRunner.runBeforeParseHook(command); @@ -514,11 +515,11 @@ public void run() { } finally { queryLifeTimeHookRunner.runAfterParseHook(command, parseError); } - perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.PARSE); + perfLogger.PerfLogEnd(CLASS_NAME, PerfLoggerTokens.PARSE); queryLifeTimeHookRunner.runBeforeCompileHook(command); - perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.ANALYZE); + perfLogger.PerfLogBegin(CLASS_NAME, PerfLoggerTokens.ANALYZE); BaseSemanticAnalyzer sem = SemanticAnalyzerFactory.get(queryState, tree); List saHooks = hooksLoader.getHooks(HiveConf.ConfVars.SEMANTIC_ANALYZER_HOOK, console); @@ -560,7 +561,7 @@ public void run() { // validate the plan sem.validate(); - perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.ANALYZE); + perfLogger.PerfLogEnd(CLASS_NAME, PerfLoggerTokens.ANALYZE); if (isInterrupted()) { return handleInterruption("after analyzing query."); @@ -568,7 +569,7 @@ public void run() { // get the output schema schema = getSchema(sem, conf); - plan = new QueryPlan(queryStr, sem, perfLogger.getStartTime(PerfLogger.DRIVER_RUN), queryId, + plan = new QueryPlan(queryStr, sem, perfLogger.getStartTime(PerfLoggerTokens.DRIVER_RUN), queryId, queryState.getHiveOperation(), schema); conf.setQueryString(queryStr); @@ -586,7 +587,7 @@ public void run() { HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_AUTHORIZATION_ENABLED)) { try { - perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.DO_AUTHORIZATION); + perfLogger.PerfLogBegin(CLASS_NAME, PerfLoggerTokens.DO_AUTHORIZATION); doAuthorization(queryState.getHiveOperation(), sem, command); } catch (AuthorizationException authExp) { console.printError("Authorization failed:" + authExp.getMessage() @@ -595,7 +596,7 @@ public void run() { SQLState = "42000"; return 403; } finally { - perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.DO_AUTHORIZATION); + perfLogger.PerfLogEnd(CLASS_NAME, PerfLoggerTokens.DO_AUTHORIZATION); } } @@ -650,7 +651,7 @@ public void run() { } } - double duration = perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.COMPILE)/1000.00; + double duration = perfLogger.PerfLogEnd(CLASS_NAME, PerfLoggerTokens.COMPILE)/1000.00; ImmutableMap compileHMSTimings = dumpMetaCallTimingWithoutEx("compilation"); queryDisplay.setHmsTimings(QueryDisplay.Phase.COMPILATION, compileHMSTimings); @@ -1171,7 +1172,7 @@ private String getUserFromUGI() { **/ private int acquireLocks() { PerfLogger perfLogger = SessionState.getPerfLogger(); - perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.ACQUIRE_READ_WRITE_LOCKS); + perfLogger.PerfLogBegin(CLASS_NAME, PerfLoggerTokens.ACQUIRE_READ_WRITE_LOCKS); SessionState ss = SessionState.get(); HiveTxnManager txnMgr = ss.getTxnMgr(); @@ -1211,7 +1212,7 @@ private int acquireLocks() { + org.apache.hadoop.util.StringUtils.stringifyException(e)); return 10; } finally { - perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.ACQUIRE_READ_WRITE_LOCKS); + perfLogger.PerfLogEnd(CLASS_NAME, PerfLoggerTokens.ACQUIRE_READ_WRITE_LOCKS); } } @@ -1228,7 +1229,7 @@ private boolean haveAcidWrite() { public void releaseLocksAndCommitOrRollback(boolean commit, HiveTxnManager txnManager) throws LockException { PerfLogger perfLogger = SessionState.getPerfLogger(); - perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.RELEASE_LOCKS); + perfLogger.PerfLogBegin(CLASS_NAME, PerfLoggerTokens.RELEASE_LOCKS); HiveTxnManager txnMgr; if (txnManager == null) { SessionState ss = SessionState.get(); @@ -1262,7 +1263,7 @@ public void releaseLocksAndCommitOrRollback(boolean commit, HiveTxnManager txnMa ctx.setHiveLocks(null); } - perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.RELEASE_LOCKS); + perfLogger.PerfLogEnd(CLASS_NAME, PerfLoggerTokens.RELEASE_LOCKS); } /** @@ -1361,10 +1362,10 @@ private int compileInternal(String command, boolean deferClose) { } PerfLogger perfLogger = SessionState.getPerfLogger(); - perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.WAIT_COMPILE); + perfLogger.PerfLogBegin(CLASS_NAME, PerfLoggerTokens.WAIT_COMPILE); final ReentrantLock compileLock = tryAcquireCompileLock(isParallelEnabled, command); - perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.WAIT_COMPILE); + perfLogger.PerfLogEnd(CLASS_NAME, PerfLoggerTokens.WAIT_COMPILE); if (metrics != null) { metrics.decrementCounter(MetricsConstant.WAITING_COMPILE_OPS, 1); } @@ -1389,8 +1390,8 @@ private int compileInternal(String command, boolean deferClose) { } //Save compile-time PerfLogging for WebUI. - //Execution-time Perf logs are done by either another thread's PerfLogger - //or a reset PerfLogger. + //Execution-time Perf logs are done by either another thread's PerfLoggerTokens + //or a reset PerfLoggerTokens. queryDisplay.setPerfLogStarts(QueryDisplay.Phase.COMPILATION, perfLogger.getStartTimes()); queryDisplay.setPerfLogEnds(QueryDisplay.Phase.COMPILATION, perfLogger.getEndTimes()); return ret; @@ -1516,7 +1517,7 @@ private CommandProcessorResponse runInternal(String command, boolean alreadyComp // reuse existing perf logger. perfLogger = SessionState.getPerfLogger(); // Since we're reusing the compiled plan, we need to update its start time for current run - plan.setQueryStartTime(perfLogger.getStartTime(PerfLogger.DRIVER_RUN)); + plan.setQueryStartTime(perfLogger.getStartTime(PerfLoggerTokens.DRIVER_RUN)); } // the reason that we set the txn manager for the cxt here is because each // query has its own ctx object. The txn mgr is shared across the @@ -1557,7 +1558,7 @@ else if(plan.getOperation() == HiveOperation.ROLLBACK) { return handleHiveException(e, 12); } - perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.DRIVER_RUN); + perfLogger.PerfLogEnd(CLASS_NAME, PerfLoggerTokens.DRIVER_RUN); queryDisplay.setPerfLogStarts(QueryDisplay.Phase.EXECUTION, perfLogger.getStartTimes()); queryDisplay.setPerfLogEnds(QueryDisplay.Phase.EXECUTION, perfLogger.getEndTimes()); @@ -1687,7 +1688,7 @@ public int execute() throws CommandNeedRetryException { public int execute(boolean deferClose) throws CommandNeedRetryException { PerfLogger perfLogger = SessionState.getPerfLogger(); - perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.DRIVER_EXECUTE); + perfLogger.PerfLogBegin(CLASS_NAME, PerfLoggerTokens.DRIVER_EXECUTE); boolean noName = StringUtils.isEmpty(conf.get(MRJobConfig.JOB_NAME)); int maxlen = conf.getIntVar(HiveConf.ConfVars.HIVEJOBNAMELENGTH); @@ -1747,18 +1748,18 @@ public int execute(boolean deferClose) throws CommandNeedRetryException { for (Hook peh : hooksLoader.getHooks(HiveConf.ConfVars.PREEXECHOOKS, console)) { if (peh instanceof ExecuteWithHookContext) { - perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.PRE_HOOK + peh.getClass().getName()); + perfLogger.PerfLogBegin(CLASS_NAME, PerfLoggerTokens.PRE_HOOK + peh.getClass().getName()); ((ExecuteWithHookContext) peh).run(hookContext); - perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.PRE_HOOK + peh.getClass().getName()); + perfLogger.PerfLogEnd(CLASS_NAME, PerfLoggerTokens.PRE_HOOK + peh.getClass().getName()); } else if (peh instanceof PreExecute) { - perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.PRE_HOOK + peh.getClass().getName()); + perfLogger.PerfLogBegin(CLASS_NAME, PerfLoggerTokens.PRE_HOOK + peh.getClass().getName()); ((PreExecute) peh).run(SessionState.get(), plan.getInputs(), plan.getOutputs(), Utils.getUGI()); - perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.PRE_HOOK + peh.getClass().getName()); + perfLogger.PerfLogEnd(CLASS_NAME, PerfLoggerTokens.PRE_HOOK + peh.getClass().getName()); } } @@ -1812,7 +1813,7 @@ public int execute(boolean deferClose) throws CommandNeedRetryException { } } - perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.RUN_TASKS); + perfLogger.PerfLogBegin(CLASS_NAME, PerfLoggerTokens.RUN_TASKS); // Loop while you either have tasks running, or tasks queued up while (driverCxt.isRunning()) { @@ -1894,7 +1895,7 @@ public int execute(boolean deferClose) throws CommandNeedRetryException { } } } - perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.RUN_TASKS); + perfLogger.PerfLogEnd(CLASS_NAME, PerfLoggerTokens.RUN_TASKS); // in case we decided to run everything in local mode, restore the // the jobtracker setting to its initial value @@ -1926,19 +1927,19 @@ public int execute(boolean deferClose) throws CommandNeedRetryException { // Get all the post execution hooks and execute them. for (Hook peh : hooksLoader.getHooks(HiveConf.ConfVars.POSTEXECHOOKS, console)) { if (peh instanceof ExecuteWithHookContext) { - perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.POST_HOOK + peh.getClass().getName()); + perfLogger.PerfLogBegin(CLASS_NAME, PerfLoggerTokens.POST_HOOK + peh.getClass().getName()); ((ExecuteWithHookContext) peh).run(hookContext); - perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.POST_HOOK + peh.getClass().getName()); + perfLogger.PerfLogEnd(CLASS_NAME, PerfLoggerTokens.POST_HOOK + peh.getClass().getName()); } else if (peh instanceof PostExecute) { - perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.POST_HOOK + peh.getClass().getName()); + perfLogger.PerfLogBegin(CLASS_NAME, PerfLoggerTokens.POST_HOOK + peh.getClass().getName()); ((PostExecute) peh).run(SessionState.get(), plan.getInputs(), plan.getOutputs(), (SessionState.get() != null ? SessionState.get().getLineageState().getLineageInfo() : null), Utils.getUGI()); - perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.POST_HOOK + peh.getClass().getName()); + perfLogger.PerfLogEnd(CLASS_NAME, PerfLoggerTokens.POST_HOOK + peh.getClass().getName()); } } @@ -1990,7 +1991,7 @@ public int execute(boolean deferClose) throws CommandNeedRetryException { if (noName) { conf.set(MRJobConfig.JOB_NAME, ""); } - double duration = perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.DRIVER_EXECUTE)/1000.00; + double duration = perfLogger.PerfLogEnd(CLASS_NAME, PerfLoggerTokens.DRIVER_EXECUTE)/1000.00; ImmutableMap executionHMSTimings = dumpMetaCallTimingWithoutEx("execution"); queryDisplay.setHmsTimings(QueryDisplay.Phase.EXECUTION, executionHMSTimings); @@ -2095,11 +2096,11 @@ private void invokeFailureHooks(PerfLogger perfLogger, hookContext.setException(exception); // Get all the failure execution hooks and execute them. for (Hook ofh : hooksLoader.getHooks(HiveConf.ConfVars.ONFAILUREHOOKS, console)) { - perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.FAILURE_HOOK + ofh.getClass().getName()); + perfLogger.PerfLogBegin(CLASS_NAME, PerfLoggerTokens.FAILURE_HOOK + ofh.getClass().getName()); ((ExecuteWithHookContext) ofh).run(hookContext); - perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.FAILURE_HOOK + ofh.getClass().getName()); + perfLogger.PerfLogEnd(CLASS_NAME, PerfLoggerTokens.FAILURE_HOOK + ofh.getClass().getName()); } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java index 07aa2ea..9a9debb 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java @@ -51,6 +51,7 @@ import org.apache.hadoop.hive.ql.exec.spark.SparkUtilities; import org.apache.hadoop.hive.ql.io.HiveKey; import org.apache.hadoop.hive.ql.log.PerfLogger; +import org.apache.hadoop.hive.ql.log.PerfLoggerTokens; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.plan.JoinCondDesc; import org.apache.hadoop.hive.ql.plan.JoinDesc; @@ -309,7 +310,7 @@ public void generateMapMetaData() throws HiveException { mapJoinTables, mapJoinTableSerdes); } - perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.LOAD_HASHTABLE); + perfLogger.PerfLogBegin(CLASS_NAME, PerfLoggerTokens.LOAD_HASHTABLE); loader.init(mapContext, mrContext, hconf, this); try { loader.load(mapJoinTables, mapJoinTableSerdes); @@ -329,7 +330,7 @@ public void generateMapMetaData() throws HiveException { = new ImmutablePair (mapJoinTables, mapJoinTableSerdes); - perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.LOAD_HASHTABLE); + perfLogger.PerfLogEnd(CLASS_NAME, PerfLoggerTokens.LOAD_HASHTABLE); if (canSkipJoinProcessing(mapContext)) { LOG.info("Skipping big table join processing for " + this.toString()); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/SerializationUtilities.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/SerializationUtilities.java index 01a652d..ec682be 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/SerializationUtilities.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/SerializationUtilities.java @@ -45,6 +45,7 @@ import org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat; import org.apache.hadoop.hive.ql.io.RCFileInputFormat; import org.apache.hadoop.hive.ql.log.PerfLogger; +import org.apache.hadoop.hive.ql.log.PerfLoggerTokens; import org.apache.hadoop.hive.ql.plan.AbstractOperatorDesc; import org.apache.hadoop.hive.ql.plan.BaseWork; import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc; @@ -476,14 +477,14 @@ private static void serializePlan(Object plan, OutputStream out, boolean cloning private static void serializePlan(Kryo kryo, Object plan, OutputStream out, boolean cloningPlan) { PerfLogger perfLogger = SessionState.getPerfLogger(); - perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.SERIALIZE_PLAN); + perfLogger.PerfLogBegin(CLASS_NAME, PerfLoggerTokens.SERIALIZE_PLAN); LOG.info("Serializing " + plan.getClass().getSimpleName() + " using kryo"); if (cloningPlan) { serializeObjectByKryo(kryo, plan, out); } else { serializeObjectByKryo(kryo, plan, out); } - perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.SERIALIZE_PLAN); + perfLogger.PerfLogEnd(CLASS_NAME, PerfLoggerTokens.SERIALIZE_PLAN); } /** @@ -515,7 +516,7 @@ private static void serializePlan(Kryo kryo, Object plan, OutputStream out, bool private static T deserializePlan(Kryo kryo, InputStream in, Class planClass, boolean cloningPlan) { PerfLogger perfLogger = SessionState.getPerfLogger(); - perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.DESERIALIZE_PLAN); + perfLogger.PerfLogBegin(CLASS_NAME, PerfLoggerTokens.DESERIALIZE_PLAN); T plan; LOG.info("Deserializing " + planClass.getSimpleName() + " using kryo"); if (cloningPlan) { @@ -523,7 +524,7 @@ private static void serializePlan(Kryo kryo, Object plan, OutputStream out, bool } else { plan = deserializeObjectByKryo(kryo, in, planClass); } - perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.DESERIALIZE_PLAN); + perfLogger.PerfLogEnd(CLASS_NAME, PerfLoggerTokens.DESERIALIZE_PLAN); return plan; } @@ -535,7 +536,7 @@ private static void serializePlan(Kryo kryo, Object plan, OutputStream out, bool public static MapredWork clonePlan(MapredWork plan) { // TODO: need proper clone. Meanwhile, let's at least keep this horror in one place PerfLogger perfLogger = SessionState.getPerfLogger(); - perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.CLONE_PLAN); + perfLogger.PerfLogBegin(CLASS_NAME, PerfLoggerTokens.CLONE_PLAN); Operator op = plan.getAnyOperator(); CompilationOpContext ctx = (op == null) ? null : op.getCompilationOpContext(); ByteArrayOutputStream baos = new ByteArrayOutputStream(4096); @@ -546,7 +547,7 @@ public static MapredWork clonePlan(MapredWork plan) { for (Operator newOp : newPlan.getAllOperators()) { newOp.setCompilationOpContext(ctx); } - perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.CLONE_PLAN); + perfLogger.PerfLogEnd(CLASS_NAME, PerfLoggerTokens.CLONE_PLAN); return newPlan; } @@ -605,7 +606,7 @@ public static MapredWork clonePlan(MapredWork plan) { */ public static BaseWork cloneBaseWork(BaseWork plan) { PerfLogger perfLogger = SessionState.getPerfLogger(); - perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.CLONE_PLAN); + perfLogger.PerfLogBegin(CLASS_NAME, PerfLoggerTokens.CLONE_PLAN); Operator op = plan.getAnyRootOperator(); CompilationOpContext ctx = (op == null) ? null : op.getCompilationOpContext(); ByteArrayOutputStream baos = new ByteArrayOutputStream(4096); @@ -616,7 +617,7 @@ public static BaseWork cloneBaseWork(BaseWork plan) { for (Operator newOp : newPlan.getAllOperators()) { newOp.setCompilationOpContext(ctx); } - perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.CLONE_PLAN); + perfLogger.PerfLogEnd(CLASS_NAME, PerfLoggerTokens.CLONE_PLAN); return newPlan; } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/SparkHashTableSinkOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/SparkHashTableSinkOperator.java index c3b1d0a..d5e7608 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/SparkHashTableSinkOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/SparkHashTableSinkOperator.java @@ -32,6 +32,7 @@ import org.apache.hadoop.hive.ql.exec.persistence.MapJoinPersistableTableContainer; import org.apache.hadoop.hive.ql.exec.persistence.MapJoinTableContainerSerDe; import org.apache.hadoop.hive.ql.log.PerfLogger; +import org.apache.hadoop.hive.ql.log.PerfLoggerTokens; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.plan.BucketMapJoinContext; import org.apache.hadoop.hive.ql.plan.MapredLocalWork; @@ -98,7 +99,7 @@ public void closeOp(boolean abort) throws HiveException { LOG.debug("Aborting, skip dumping side-table for tag: " + tag); } } else { - String method = PerfLogger.SPARK_FLUSH_HASHTABLE + getName(); + String method = PerfLoggerTokens.SPARK_FLUSH_HASHTABLE + getName(); perfLogger.PerfLogBegin(CLASS_NAME, method); try { flushToFile(mapJoinTables[tag], tag); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java index 9036d9e..fbaa0b5 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java @@ -87,6 +87,7 @@ import org.apache.hadoop.hive.ql.io.rcfile.truncate.ColumnTruncateMapper; import org.apache.hadoop.hive.ql.io.rcfile.truncate.ColumnTruncateWork; import org.apache.hadoop.hive.ql.log.PerfLogger; +import org.apache.hadoop.hive.ql.log.PerfLoggerTokens; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.metadata.HiveStorageHandler; import org.apache.hadoop.hive.ql.metadata.HiveUtils; @@ -2124,7 +2125,7 @@ static int getMaxExecutorsForInputListing(final Configuration conf, int inputLoc public static ContentSummary getInputSummary(final Context ctx, MapWork work, PathFilter filter) throws IOException { PerfLogger perfLogger = SessionState.getPerfLogger(); - perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.INPUT_SUMMARY); + perfLogger.PerfLogBegin(CLASS_NAME, PerfLoggerTokens.INPUT_SUMMARY); long[] summary = {0, 0, 0}; @@ -2298,7 +2299,7 @@ public void run() { + cs.getFileCount() + " directory count: " + cs.getDirectoryCount()); } - perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.INPUT_SUMMARY); + perfLogger.PerfLogEnd(CLASS_NAME, PerfLoggerTokens.INPUT_SUMMARY); return new ContentSummary(summary[0], summary[1], summary[2]); } finally { HiveInterruptUtils.remove(interrup); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkMapRecordHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkMapRecordHandler.java index 48dfedc..e107a39 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkMapRecordHandler.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkMapRecordHandler.java @@ -34,7 +34,7 @@ import org.apache.hadoop.hive.ql.exec.mr.ExecMapper.ReportStats; import org.apache.hadoop.hive.ql.exec.mr.ExecMapperContext; import org.apache.hadoop.hive.ql.exec.vector.VectorMapOperator; -import org.apache.hadoop.hive.ql.log.PerfLogger; +import org.apache.hadoop.hive.ql.log.PerfLoggerTokens; import org.apache.hadoop.hive.ql.plan.MapWork; import org.apache.hadoop.hive.ql.plan.MapredLocalWork; import org.apache.hadoop.hive.ql.plan.OperatorDesc; @@ -63,7 +63,7 @@ @Override public void init(JobConf job, OutputCollector output, Reporter reporter) throws Exception { - perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.SPARK_INIT_OPERATORS); + perfLogger.PerfLogBegin(CLASS_NAME, PerfLoggerTokens.SPARK_INIT_OPERATORS); super.init(job, output, reporter); isLogInfoEnabled = LOG.isInfoEnabled(); @@ -122,7 +122,7 @@ throw new RuntimeException("Map operator initialization failed: " + e, e); } } - perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.SPARK_INIT_OPERATORS); + perfLogger.PerfLogEnd(CLASS_NAME, PerfLoggerTokens.SPARK_INIT_OPERATORS); } @Override diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkPlan.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkPlan.java index 9a2ab51..79150b1 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkPlan.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkPlan.java @@ -30,6 +30,7 @@ import org.slf4j.LoggerFactory; import org.apache.hadoop.hive.ql.io.HiveKey; import org.apache.hadoop.hive.ql.log.PerfLogger; +import org.apache.hadoop.hive.ql.log.PerfLoggerTokens; import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hadoop.io.BytesWritable; import org.apache.spark.api.java.JavaPairRDD; @@ -50,7 +51,7 @@ @SuppressWarnings("unchecked") public JavaPairRDD generateGraph() { - perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.SPARK_BUILD_RDD_GRAPH); + perfLogger.PerfLogBegin(CLASS_NAME, PerfLoggerTokens.SPARK_BUILD_RDD_GRAPH); Map> tranToOutputRDDMap = new HashMap>(); for (SparkTran tran : getAllTrans()) { @@ -88,7 +89,7 @@ } } - perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.SPARK_BUILD_RDD_GRAPH); + perfLogger.PerfLogEnd(CLASS_NAME, PerfLoggerTokens.SPARK_BUILD_RDD_GRAPH); if (LOG.isDebugEnabled()) { LOG.info("print generated spark rdd graph:\n" + SparkUtilities.rddGraphToString(finalRDD)); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkPlanGenerator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkPlanGenerator.java index 5f85f9e..a3a8573 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkPlanGenerator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkPlanGenerator.java @@ -34,6 +34,7 @@ import org.apache.hadoop.hive.ql.io.merge.MergeFileOutputFormat; import org.apache.hadoop.hive.ql.io.merge.MergeFileWork; import org.apache.hadoop.hive.ql.log.PerfLogger; +import org.apache.hadoop.hive.ql.log.PerfLoggerTokens; import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hadoop.mapred.FileOutputFormat; import org.apache.hadoop.hive.conf.HiveConf; @@ -97,7 +98,7 @@ public SparkPlanGenerator( } public SparkPlan generate(SparkWork sparkWork) throws Exception { - perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.SPARK_BUILD_PLAN); + perfLogger.PerfLogBegin(CLASS_NAME, PerfLoggerTokens.SPARK_BUILD_PLAN); SparkPlan sparkPlan = new SparkPlan(); cloneToWork = sparkWork.getCloneToWork(); workToTranMap.clear(); @@ -105,13 +106,13 @@ public SparkPlan generate(SparkWork sparkWork) throws Exception { try { for (BaseWork work : sparkWork.getAllWork()) { - perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.SPARK_CREATE_TRAN + work.getName()); + perfLogger.PerfLogBegin(CLASS_NAME, PerfLoggerTokens.SPARK_CREATE_TRAN + work.getName()); SparkTran tran = generate(work, sparkWork); SparkTran parentTran = generateParentTran(sparkPlan, sparkWork, work); sparkPlan.addTran(tran); sparkPlan.connect(parentTran, tran); workToTranMap.put(work, tran); - perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.SPARK_CREATE_TRAN + work.getName()); + perfLogger.PerfLogEnd(CLASS_NAME, PerfLoggerTokens.SPARK_CREATE_TRAN + work.getName()); } } finally { // clear all ThreadLocal cached MapWork/ReduceWork after plan generation @@ -119,7 +120,7 @@ public SparkPlan generate(SparkWork sparkWork) throws Exception { Utilities.clearWorkMap(jobConf); } - perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.SPARK_BUILD_PLAN); + perfLogger.PerfLogEnd(CLASS_NAME, PerfLoggerTokens.SPARK_BUILD_PLAN); return sparkPlan; } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkRecordHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkRecordHandler.java index 2421885..8c52946 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkRecordHandler.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkRecordHandler.java @@ -22,6 +22,7 @@ import org.slf4j.LoggerFactory; import org.apache.hadoop.hive.ql.exec.MapredContext; import org.apache.hadoop.hive.ql.log.PerfLogger; +import org.apache.hadoop.hive.ql.log.PerfLoggerTokens; import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.OutputCollector; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkReduceRecordHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkReduceRecordHandler.java index 7eaad18..4081f36 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkReduceRecordHandler.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkReduceRecordHandler.java @@ -36,7 +36,7 @@ import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpressionWriter; import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpressionWriterFactory; -import org.apache.hadoop.hive.ql.log.PerfLogger; +import org.apache.hadoop.hive.ql.log.PerfLoggerTokens; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.plan.MapredLocalWork; import org.apache.hadoop.hive.ql.plan.OperatorDesc; @@ -105,7 +105,7 @@ @Override @SuppressWarnings("unchecked") public void init(JobConf job, OutputCollector output, Reporter reporter) throws Exception { - perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.SPARK_INIT_OPERATORS); + perfLogger.PerfLogBegin(CLASS_NAME, PerfLoggerTokens.SPARK_INIT_OPERATORS); super.init(job, output, reporter); rowObjectInspector = new ObjectInspector[Byte.MAX_VALUE]; @@ -208,7 +208,7 @@ public void init(JobConf job, OutputCollector output, Reporter reporter) throws throw new RuntimeException("Reduce operator initialization failed", e); } } - perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.SPARK_INIT_OPERATORS); + perfLogger.PerfLogEnd(CLASS_NAME, PerfLoggerTokens.SPARK_INIT_OPERATORS); } /** diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkTask.java index b4fb49f..8426471 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkTask.java @@ -55,6 +55,7 @@ import org.apache.hadoop.hive.ql.exec.spark.status.SparkStageProgress; import org.apache.hadoop.hive.ql.history.HiveHistory.Keys; import org.apache.hadoop.hive.ql.log.PerfLogger; +import org.apache.hadoop.hive.ql.log.PerfLoggerTokens; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.plan.BaseWork; import org.apache.hadoop.hive.ql.plan.MapWork; @@ -108,10 +109,10 @@ public int execute(DriverContext driverContext) { SparkWork sparkWork = getWork(); sparkWork.setRequiredCounterPrefix(getOperatorCounters()); - perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.SPARK_SUBMIT_JOB); - submitTime = perfLogger.getStartTime(PerfLogger.SPARK_SUBMIT_JOB); + perfLogger.PerfLogBegin(CLASS_NAME, PerfLoggerTokens.SPARK_SUBMIT_JOB); + submitTime = perfLogger.getStartTime(PerfLoggerTokens.SPARK_SUBMIT_JOB); jobRef = sparkSession.submit(driverContext, sparkWork); - perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.SPARK_SUBMIT_JOB); + perfLogger.PerfLogEnd(CLASS_NAME, PerfLoggerTokens.SPARK_SUBMIT_JOB); if (driverContext.isShutdown()) { killJob(); @@ -157,7 +158,7 @@ public int execute(DriverContext driverContext) { setException(e); rc = 1; } finally { - startTime = perfLogger.getEndTime(PerfLogger.SPARK_SUBMIT_TO_RUNNING); + startTime = perfLogger.getEndTime(PerfLoggerTokens.SPARK_SUBMIT_TO_RUNNING); // The startTime may not be set if the sparkTask finished too fast, // because SparkJobMonitor will sleep for 1 second then check the state, // right after sleep, the spark job may be already completed. @@ -165,7 +166,7 @@ public int execute(DriverContext driverContext) { if (startTime < submitTime) { startTime = submitTime; } - finishTime = perfLogger.getEndTime(PerfLogger.SPARK_RUN_JOB); + finishTime = perfLogger.getEndTime(PerfLoggerTokens.SPARK_RUN_JOB); Utilities.clearWork(conf); if (sparkSession != null && sparkSessionManager != null) { rc = close(rc); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/LocalSparkJobMonitor.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/LocalSparkJobMonitor.java index a678228..eaa3f3b 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/LocalSparkJobMonitor.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/LocalSparkJobMonitor.java @@ -21,7 +21,7 @@ import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.exec.Utilities; -import org.apache.hadoop.hive.ql.log.PerfLogger; +import org.apache.hadoop.hive.ql.log.PerfLoggerTokens; import org.apache.spark.JobExecutionStatus; /** @@ -44,8 +44,8 @@ public int startMonitor() { JobExecutionStatus lastState = null; Map lastProgressMap = null; - perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.SPARK_RUN_JOB); - perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.SPARK_SUBMIT_TO_RUNNING); + perfLogger.PerfLogBegin(CLASS_NAME, PerfLoggerTokens.SPARK_RUN_JOB); + perfLogger.PerfLogBegin(CLASS_NAME, PerfLoggerTokens.SPARK_SUBMIT_TO_RUNNING); startTime = System.currentTimeMillis(); @@ -73,7 +73,7 @@ public int startMonitor() { switch (state) { case RUNNING: if (!running) { - perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.SPARK_SUBMIT_TO_RUNNING); + perfLogger.PerfLogEnd(CLASS_NAME, PerfLoggerTokens.SPARK_SUBMIT_TO_RUNNING); // print job stages. console.printInfo("\nQuery Hive on Spark job[" + sparkJobStatus.getJobId() + "] stages:"); @@ -136,7 +136,7 @@ public int startMonitor() { } } - perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.SPARK_RUN_JOB); + perfLogger.PerfLogEnd(CLASS_NAME, PerfLoggerTokens.SPARK_RUN_JOB); return rc; } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/RemoteSparkJobMonitor.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/RemoteSparkJobMonitor.java index 9dfb65e..9fe8c31 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/RemoteSparkJobMonitor.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/RemoteSparkJobMonitor.java @@ -25,7 +25,7 @@ import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.exec.Utilities; import org.apache.hadoop.hive.ql.exec.spark.status.impl.RemoteSparkJobStatus; -import org.apache.hadoop.hive.ql.log.PerfLogger; +import org.apache.hadoop.hive.ql.log.PerfLoggerTokens; import org.apache.hive.spark.client.JobHandle; import org.apache.spark.JobExecutionStatus; @@ -53,8 +53,8 @@ public int startMonitor() { int rc = 0; Map lastProgressMap = null; - perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.SPARK_RUN_JOB); - perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.SPARK_SUBMIT_TO_RUNNING); + perfLogger.PerfLogBegin(CLASS_NAME, PerfLoggerTokens.SPARK_RUN_JOB); + perfLogger.PerfLogBegin(CLASS_NAME, PerfLoggerTokens.SPARK_SUBMIT_TO_RUNNING); startTime = System.currentTimeMillis(); @@ -85,7 +85,7 @@ public int startMonitor() { if (sparkJobState == JobExecutionStatus.RUNNING) { Map progressMap = sparkJobStatus.getSparkStageProgress(); if (!running) { - perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.SPARK_SUBMIT_TO_RUNNING); + perfLogger.PerfLogEnd(CLASS_NAME, PerfLoggerTokens.SPARK_SUBMIT_TO_RUNNING); printAppInfo(); // print job stages. console.printInfo("\nQuery Hive on Spark job[" + sparkJobStatus.getJobId() + @@ -184,7 +184,7 @@ public int startMonitor() { } } - perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.SPARK_RUN_JOB); + perfLogger.PerfLogEnd(CLASS_NAME, PerfLoggerTokens.SPARK_RUN_JOB); return rc; } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/SparkJobMonitor.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/SparkJobMonitor.java index 41730b5..101655f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/SparkJobMonitor.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/SparkJobMonitor.java @@ -21,6 +21,7 @@ import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.common.log.InPlaceUpdate; import org.apache.hadoop.hive.ql.log.PerfLogger; +import org.apache.hadoop.hive.ql.log.PerfLoggerTokens; import org.apache.hadoop.hive.ql.session.SessionState; import org.fusesource.jansi.Ansi; import org.slf4j.Logger; @@ -118,14 +119,14 @@ private void printStatusInPlace(Map progressMap) { StageState state = total > 0 ? StageState.PENDING : StageState.FINISHED; if (complete > 0 || running > 0 || failed > 0) { - if (!perfLogger.startTimeHasMethod(PerfLogger.SPARK_RUN_STAGE + s)) { - perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.SPARK_RUN_STAGE + s); + if (!perfLogger.startTimeHasMethod(PerfLoggerTokens.SPARK_RUN_STAGE + s)) { + perfLogger.PerfLogBegin(CLASS_NAME, PerfLoggerTokens.SPARK_RUN_STAGE + s); } if (complete < total) { state = StageState.RUNNING; } else { state = StageState.FINISHED; - perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.SPARK_RUN_STAGE + s); + perfLogger.PerfLogEnd(CLASS_NAME, PerfLoggerTokens.SPARK_RUN_STAGE + s); completed.add(s); } } @@ -208,15 +209,15 @@ private String getReport(Map progressMap) { if (complete == total && !completed.contains(s)) { completed.add(s); - if (!perfLogger.startTimeHasMethod(PerfLogger.SPARK_RUN_STAGE + s)) { - perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.SPARK_RUN_STAGE + s); + if (!perfLogger.startTimeHasMethod(PerfLoggerTokens.SPARK_RUN_STAGE + s)) { + perfLogger.PerfLogBegin(CLASS_NAME, PerfLoggerTokens.SPARK_RUN_STAGE + s); } - perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.SPARK_RUN_STAGE + s); + perfLogger.PerfLogEnd(CLASS_NAME, PerfLoggerTokens.SPARK_RUN_STAGE + s); } if (complete < total && (complete > 0 || running > 0 || failed > 0)) { /* stage is started, but not complete */ - if (!perfLogger.startTimeHasMethod(PerfLogger.SPARK_RUN_STAGE + s)) { - perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.SPARK_RUN_STAGE + s); + if (!perfLogger.startTimeHasMethod(PerfLoggerTokens.SPARK_RUN_STAGE + s)) { + perfLogger.PerfLogBegin(CLASS_NAME, PerfLoggerTokens.SPARK_RUN_STAGE + s); } if (failed > 0) { reportBuffer.append( diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/MapRecordProcessor.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/MapRecordProcessor.java index 24d3526..8877e2b 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/MapRecordProcessor.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/MapRecordProcessor.java @@ -55,7 +55,7 @@ import org.apache.hadoop.hive.ql.exec.tez.TezProcessor.TezKVOutputCollector; import org.apache.hadoop.hive.ql.exec.tez.tools.KeyValueInputMerger; import org.apache.hadoop.hive.ql.exec.vector.VectorMapOperator; -import org.apache.hadoop.hive.ql.log.PerfLogger; +import org.apache.hadoop.hive.ql.log.PerfLoggerTokens; import org.apache.hadoop.hive.ql.plan.BaseWork; import org.apache.hadoop.hive.ql.plan.DynamicValue; import org.apache.hadoop.hive.ql.plan.MapWork; @@ -119,7 +119,7 @@ private void setLlapOfFragmentId(final ProcessorContext context) { @Override void init(MRTaskReporter mrReporter, Map inputs, Map outputs) throws Exception { - perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.TEZ_INIT_OPERATORS); + perfLogger.PerfLogBegin(CLASS_NAME, PerfLoggerTokens.TEZ_INIT_OPERATORS); super.init(mrReporter, inputs, outputs); checkAbortCondition(); @@ -354,7 +354,7 @@ public DynamicValueRegistryTez call() { throw new RuntimeException("Map operator initialization failed", e); } } - perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.TEZ_INIT_OPERATORS); + perfLogger.PerfLogEnd(CLASS_NAME, PerfLoggerTokens.TEZ_INIT_OPERATORS); } private void initializeMapRecordSources() throws Exception { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/MergeFileRecordProcessor.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/MergeFileRecordProcessor.java index 69de3a0..d74bb1a 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/MergeFileRecordProcessor.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/MergeFileRecordProcessor.java @@ -33,7 +33,7 @@ import org.apache.hadoop.hive.ql.exec.mr.ExecMapper; import org.apache.hadoop.hive.ql.exec.mr.ExecMapperContext; import org.apache.hadoop.hive.ql.io.merge.MergeFileWork; -import org.apache.hadoop.hive.ql.log.PerfLogger; +import org.apache.hadoop.hive.ql.log.PerfLoggerTokens; import org.apache.hadoop.hive.ql.plan.MapWork; import org.apache.hadoop.hive.ql.plan.OperatorDesc; import org.apache.hadoop.mapred.JobConf; @@ -74,7 +74,7 @@ void init( MRTaskReporter mrReporter, Map inputs, Map outputs) throws Exception { // TODO HIVE-14042. Abort handling. - perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.TEZ_INIT_OPERATORS); + perfLogger.PerfLogBegin(CLASS_NAME, PerfLoggerTokens.TEZ_INIT_OPERATORS); super.init(mrReporter, inputs, outputs); execContext = new ExecMapperContext(jconf); @@ -142,7 +142,7 @@ public Object call() { throw new RuntimeException("Map operator initialization failed", e); } } - perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.TEZ_INIT_OPERATORS); + perfLogger.PerfLogEnd(CLASS_NAME, PerfLoggerTokens.TEZ_INIT_OPERATORS); } @Override diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/RecordProcessor.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/RecordProcessor.java index 106a534..69c4da9 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/RecordProcessor.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/RecordProcessor.java @@ -28,6 +28,7 @@ import org.apache.hadoop.hive.ql.exec.Utilities; import org.apache.hadoop.hive.ql.exec.tez.TezProcessor.TezKVOutputCollector; import org.apache.hadoop.hive.ql.log.PerfLogger; +import org.apache.hadoop.hive.ql.log.PerfLoggerTokens; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.plan.BaseWork; import org.apache.hadoop.hive.ql.session.SessionState; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ReduceRecordProcessor.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ReduceRecordProcessor.java index f6f2dd0..682d6ee 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ReduceRecordProcessor.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ReduceRecordProcessor.java @@ -42,7 +42,7 @@ import org.apache.hadoop.hive.ql.exec.mr.ExecMapper.ReportStats; import org.apache.hadoop.hive.ql.exec.tez.DynamicValueRegistryTez.RegistryConfTez; import org.apache.hadoop.hive.ql.exec.tez.TezProcessor.TezKVOutputCollector; -import org.apache.hadoop.hive.ql.log.PerfLogger; +import org.apache.hadoop.hive.ql.log.PerfLoggerTokens; import org.apache.hadoop.hive.ql.plan.BaseWork; import org.apache.hadoop.hive.ql.plan.DynamicValue; import org.apache.hadoop.hive.ql.plan.ReduceWork; @@ -110,7 +110,7 @@ public Object call() { void init( MRTaskReporter mrReporter, Map inputs, Map outputs) throws Exception { - perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.TEZ_INIT_OPERATORS); + perfLogger.PerfLogBegin(CLASS_NAME, PerfLoggerTokens.TEZ_INIT_OPERATORS); super.init(mrReporter, inputs, outputs); MapredContext.init(false, new JobConf(jconf)); @@ -266,7 +266,7 @@ public DynamicValueRegistryTez call() { } } - perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.TEZ_INIT_OPERATORS); + perfLogger.PerfLogEnd(CLASS_NAME, PerfLoggerTokens.TEZ_INIT_OPERATORS); } private void initializeMultipleSources(ReduceWork redWork, int numTags, ObjectInspector[] ois, diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ReduceRecordSource.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ReduceRecordSource.java index 60660ac..a836d48 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ReduceRecordSource.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ReduceRecordSource.java @@ -37,6 +37,7 @@ import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpressionWriter; import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpressionWriterFactory; import org.apache.hadoop.hive.ql.log.PerfLogger; +import org.apache.hadoop.hive.ql.log.PerfLoggerTokens; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.plan.TableDesc; import org.apache.hadoop.hive.ql.session.SessionState; @@ -232,7 +233,7 @@ void init(JobConf jconf, Operator reducer, boolean vectorized, TableDesc keyT throw new RuntimeException("Reduce operator initialization failed", e); } } - perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.TEZ_INIT_OPERATORS); + perfLogger.PerfLogEnd(CLASS_NAME, PerfLoggerTokens.TEZ_INIT_OPERATORS); } @Override diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezProcessor.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezProcessor.java index 4242262..82e4ed4 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezProcessor.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezProcessor.java @@ -29,6 +29,7 @@ import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.ql.log.PerfLogger; +import org.apache.hadoop.hive.ql.log.PerfLoggerTokens; import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.OutputCollector; @@ -100,7 +101,7 @@ public void handleEvents(List arg0) { @Override public void initialize() throws IOException { - perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.TEZ_INITIALIZE_PROCESSOR); + perfLogger.PerfLogBegin(CLASS_NAME, PerfLoggerTokens.TEZ_INITIALIZE_PROCESSOR); Configuration conf = TezUtils.createConfFromUserPayload(getContext().getUserPayload()); this.jobConf = new JobConf(conf); this.processorContext = getContext(); @@ -109,7 +110,7 @@ public void initialize() throws IOException { ((Hook)execCtx).initializeHook(this); } setupMRLegacyConfigs(processorContext); - perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.TEZ_INITIALIZE_PROCESSOR); + perfLogger.PerfLogEnd(CLASS_NAME, PerfLoggerTokens.TEZ_INITIALIZE_PROCESSOR); } private void setupMRLegacyConfigs(ProcessorContext processorContext) { @@ -144,7 +145,7 @@ public void run(Map inputs, Map out return; } - perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.TEZ_RUN_PROCESSOR); + perfLogger.PerfLogBegin(CLASS_NAME, PerfLoggerTokens.TEZ_RUN_PROCESSOR); // in case of broadcast-join read the broadcast edge inputs // (possibly asynchronously) @@ -189,7 +190,7 @@ protected void initializeAndRunProcessor(Map inputs, rproc.run(); //done - output does not need to be committed as hive does not use outputcommitter - perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.TEZ_RUN_PROCESSOR); + perfLogger.PerfLogEnd(CLASS_NAME, PerfLoggerTokens.TEZ_RUN_PROCESSOR); } catch (Throwable t) { originalThrowable = t; } finally { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java index 1c84c6a..7cadddd 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java @@ -49,6 +49,7 @@ import org.apache.hadoop.hive.ql.exec.Task; import org.apache.hadoop.hive.ql.exec.Utilities; import org.apache.hadoop.hive.ql.log.PerfLogger; +import org.apache.hadoop.hive.ql.log.PerfLoggerTokens; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.plan.BaseWork; import org.apache.hadoop.hive.ql.plan.MapWork; @@ -340,7 +341,7 @@ DAG build(JobConf conf, TezWork work, Path scratchDir, LocalResource appJarLr, List additionalLr, Context ctx) throws Exception { - perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.TEZ_BUILD_DAG); + perfLogger.PerfLogBegin(CLASS_NAME, PerfLoggerTokens.TEZ_BUILD_DAG); // getAllWork returns a topologically sorted list, which we use to make // sure that vertices are created before they are used in edges. @@ -372,7 +373,7 @@ DAG build(JobConf conf, TezWork work, Path scratchDir, boolean isFinal = work.getLeaves().contains(w); // translate work to vertex - perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.TEZ_CREATE_VERTEX + w.getName()); + perfLogger.PerfLogBegin(CLASS_NAME, PerfLoggerTokens.TEZ_CREATE_VERTEX + w.getName()); if (w instanceof UnionWork) { // Special case for unions. These items translate to VertexGroups @@ -429,7 +430,7 @@ DAG build(JobConf conf, TezWork work, Path scratchDir, } // Otherwise just leave it up to Tez to decide how much memory to allocate dag.addVertex(wx); utils.addCredentials(w, dag); - perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.TEZ_CREATE_VERTEX + w.getName()); + perfLogger.PerfLogEnd(CLASS_NAME, PerfLoggerTokens.TEZ_CREATE_VERTEX + w.getName()); workToVertex.put(w, wx); workToConf.put(w, wxConf); @@ -447,7 +448,7 @@ DAG build(JobConf conf, TezWork work, Path scratchDir, } // Clear the work map after build. TODO: remove caching instead? Utilities.clearWorkMap(conf); - perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.TEZ_BUILD_DAG); + perfLogger.PerfLogEnd(CLASS_NAME, PerfLoggerTokens.TEZ_BUILD_DAG); return dag; } @@ -485,7 +486,7 @@ DAGClient submit(JobConf conf, DAG dag, Path scratchDir, Map inputOutputLocalResources) throws Exception { - perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.TEZ_SUBMIT_DAG); + perfLogger.PerfLogBegin(CLASS_NAME, PerfLoggerTokens.TEZ_SUBMIT_DAG); DAGClient dagClient = null; Map resourceMap = new HashMap(); @@ -530,7 +531,7 @@ DAGClient submit(JobConf conf, DAG dag, Path scratchDir, } } - perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.TEZ_SUBMIT_DAG); + perfLogger.PerfLogEnd(CLASS_NAME, PerfLoggerTokens.TEZ_SUBMIT_DAG); return new SyncDagClient(dagClient); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/monitoring/DAGSummary.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/monitoring/DAGSummary.java index 7cb74a5..a63c9ec 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/monitoring/DAGSummary.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/monitoring/DAGSummary.java @@ -23,12 +23,12 @@ import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator; import org.apache.hadoop.hive.common.log.InPlaceUpdate; import org.apache.hadoop.hive.ql.log.PerfLogger; +import org.apache.hadoop.hive.ql.log.PerfLoggerTokens; import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.tez.common.counters.TaskCounter; import org.apache.tez.common.counters.TezCounter; import org.apache.tez.common.counters.TezCounters; import org.apache.tez.dag.api.DAG; -import org.apache.tez.dag.api.TezConfiguration; import org.apache.tez.dag.api.TezException; import org.apache.tez.dag.api.Vertex; import org.apache.tez.dag.api.client.DAGClient; @@ -213,7 +213,7 @@ private String vertexSummary(String vertexName, Progress progress, VertexStatus hiveCounterValue(formattedName(FileSinkOperator.Counter.RECORDS_OUT.toString(), vertexName)) + hiveCounterValue(formattedName(ReduceSinkOperator.Counter.RECORDS_OUT_INTERMEDIATE.toString(), vertexName)); - final double duration = perfLogger.getDuration(PerfLogger.TEZ_RUN_VERTEX + vertexName); + final double duration = perfLogger.getDuration(PerfLoggerTokens.TEZ_RUN_VERTEX + vertexName); return String.format(FORMATTING_PATTERN, vertexName, diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/monitoring/QueryExecutionBreakdownSummary.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/monitoring/QueryExecutionBreakdownSummary.java index 271e3c6..2fafdab 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/monitoring/QueryExecutionBreakdownSummary.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/monitoring/QueryExecutionBreakdownSummary.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hive.ql.exec.tez.monitoring; import org.apache.hadoop.hive.ql.log.PerfLogger; +import org.apache.hadoop.hive.ql.log.PerfLoggerTokens; import org.apache.hadoop.hive.ql.session.SessionState; import java.text.DecimalFormat; @@ -40,9 +41,9 @@ QueryExecutionBreakdownSummary(PerfLogger perfLogger) { this.perfLogger = perfLogger; - this.compileEndTime = perfLogger.getEndTime(PerfLogger.COMPILE); - this.dagSubmitStartTime = perfLogger.getStartTime(PerfLogger.TEZ_SUBMIT_DAG); - this.submitToRunningDuration = perfLogger.getDuration(PerfLogger.TEZ_SUBMIT_TO_RUNNING); + this.compileEndTime = perfLogger.getEndTime(PerfLoggerTokens.COMPILE); + this.dagSubmitStartTime = perfLogger.getStartTime(PerfLoggerTokens.TEZ_SUBMIT_DAG); + this.submitToRunningDuration = perfLogger.getDuration(PerfLoggerTokens.TEZ_SUBMIT_TO_RUNNING); } private String formatNumber(long number) { @@ -62,7 +63,7 @@ public void print(SessionState.LogHelper console) { console.printInfo(SEPARATOR); // parse, analyze, optimize and compile - long compile = compileEndTime - perfLogger.getStartTime(PerfLogger.COMPILE); + long compile = compileEndTime - perfLogger.getStartTime(PerfLoggerTokens.COMPILE); console.printInfo(format("Compile Query", compile)); // prepare plan for submission (building DAG, adding resources, creating scratch dirs etc.) @@ -71,7 +72,7 @@ public void print(SessionState.LogHelper console) { // submit to accept dag (if session is closed, this will include re-opening of session time, // localizing files for AM, submitting DAG) - long submitToAccept = perfLogger.getStartTime(PerfLogger.TEZ_RUN_DAG) - dagSubmitStartTime; + long submitToAccept = perfLogger.getStartTime(PerfLoggerTokens.TEZ_RUN_DAG) - dagSubmitStartTime; console.printInfo(format("Submit Plan", submitToAccept)); // accept to start dag (schedule wait time, resource wait time etc.) @@ -80,10 +81,10 @@ public void print(SessionState.LogHelper console) { // time to actually run the dag (actual dag runtime) final long startToEnd; if (submitToRunningDuration == 0) { - startToEnd = perfLogger.getDuration(PerfLogger.TEZ_RUN_DAG); + startToEnd = perfLogger.getDuration(PerfLoggerTokens.TEZ_RUN_DAG); } else { - startToEnd = perfLogger.getEndTime(PerfLogger.TEZ_RUN_DAG) - - perfLogger.getEndTime(PerfLogger.TEZ_SUBMIT_TO_RUNNING); + startToEnd = perfLogger.getEndTime(PerfLoggerTokens.TEZ_RUN_DAG) - + perfLogger.getEndTime(PerfLoggerTokens.TEZ_SUBMIT_TO_RUNNING); } console.printInfo(format("Run DAG", startToEnd)); console.printInfo(SEPARATOR); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/monitoring/RenderStrategy.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/monitoring/RenderStrategy.java index 3aebbe1..37441f0 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/monitoring/RenderStrategy.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/monitoring/RenderStrategy.java @@ -21,6 +21,7 @@ import org.apache.hadoop.hive.common.log.ProgressMonitor; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.log.PerfLogger; +import org.apache.hadoop.hive.ql.log.PerfLoggerTokens; import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.tez.dag.api.client.DAGStatus; import org.apache.tez.dag.api.client.Progress; @@ -92,18 +93,18 @@ private String getReport(Map progressMap) { /* * We may have missed the start of the vertex due to the 3 seconds interval */ - if (!perfLogger.startTimeHasMethod(PerfLogger.TEZ_RUN_VERTEX + s)) { - perfLogger.PerfLogBegin(TezJobMonitor.CLASS_NAME, PerfLogger.TEZ_RUN_VERTEX + s); + if (!perfLogger.startTimeHasMethod(PerfLoggerTokens.TEZ_RUN_VERTEX + s)) { + perfLogger.PerfLogBegin(TezJobMonitor.CLASS_NAME, PerfLoggerTokens.TEZ_RUN_VERTEX + s); } - if (!perfLogger.endTimeHasMethod(PerfLogger.TEZ_RUN_VERTEX + s)) { - perfLogger.PerfLogEnd(TezJobMonitor.CLASS_NAME, PerfLogger.TEZ_RUN_VERTEX + s); + if (!perfLogger.endTimeHasMethod(PerfLoggerTokens.TEZ_RUN_VERTEX + s)) { + perfLogger.PerfLogEnd(TezJobMonitor.CLASS_NAME, PerfLoggerTokens.TEZ_RUN_VERTEX + s); } } if (complete < total && (complete > 0 || running > 0 || failed > 0)) { - if (!perfLogger.startTimeHasMethod(PerfLogger.TEZ_RUN_VERTEX + s)) { - perfLogger.PerfLogBegin(TezJobMonitor.CLASS_NAME, PerfLogger.TEZ_RUN_VERTEX + s); + if (!perfLogger.startTimeHasMethod(PerfLoggerTokens.TEZ_RUN_VERTEX + s)) { + perfLogger.PerfLogBegin(TezJobMonitor.CLASS_NAME, PerfLoggerTokens.TEZ_RUN_VERTEX + s); } /* vertex is started, but not complete */ diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/monitoring/TezJobMonitor.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/monitoring/TezJobMonitor.java index f2f97f3..6b6b697 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/monitoring/TezJobMonitor.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/monitoring/TezJobMonitor.java @@ -26,6 +26,7 @@ Licensed to the Apache Software Foundation (ASF) under one import org.apache.hadoop.hive.ql.exec.tez.TezSessionPoolManager; import org.apache.hadoop.hive.common.log.InPlaceUpdate; import org.apache.hadoop.hive.ql.log.PerfLogger; +import org.apache.hadoop.hive.ql.log.PerfLoggerTokens; import org.apache.hadoop.hive.common.log.ProgressMonitor; import org.apache.hadoop.hive.ql.plan.BaseWork; import org.apache.hadoop.hive.ql.session.SessionState; @@ -133,8 +134,8 @@ public int monitorExecution() { synchronized (shutdownList) { shutdownList.add(dagClient); } - perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.TEZ_RUN_DAG); - perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.TEZ_SUBMIT_TO_RUNNING); + perfLogger.PerfLogBegin(CLASS_NAME, PerfLoggerTokens.TEZ_RUN_DAG); + perfLogger.PerfLogBegin(CLASS_NAME, PerfLoggerTokens.TEZ_SUBMIT_TO_RUNNING); DAGStatus.State lastState = null; boolean running = false; @@ -162,7 +163,7 @@ public int monitorExecution() { break; case RUNNING: if (!running) { - perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.TEZ_SUBMIT_TO_RUNNING); + perfLogger.PerfLogEnd(CLASS_NAME, PerfLoggerTokens.TEZ_SUBMIT_TO_RUNNING); console.printInfo("Status: Running (" + dagClient.getExecutionContext() + ")\n"); this.executionStartTime = System.currentTimeMillis(); running = true; @@ -234,7 +235,7 @@ public int monitorExecution() { } } - perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.TEZ_RUN_DAG); + perfLogger.PerfLogEnd(CLASS_NAME, PerfLoggerTokens.TEZ_RUN_DAG); printSummary(success, vertexProgressMap); return rc; } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/hooks/ATSHook.java b/ql/src/java/org/apache/hadoop/hive/ql/hooks/ATSHook.java index f44661e..c512643 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/hooks/ATSHook.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/hooks/ATSHook.java @@ -41,7 +41,7 @@ import org.apache.hadoop.hive.ql.exec.TaskFactory; import org.apache.hadoop.hive.ql.exec.Utilities; import org.apache.hadoop.hive.ql.exec.tez.TezTask; -import org.apache.hadoop.hive.ql.log.PerfLogger; +import org.apache.hadoop.hive.ql.log.PerfLoggerTokens; import org.apache.hadoop.hive.ql.parse.ExplainConfiguration; import org.apache.hadoop.hive.ql.plan.ExplainWork; import org.apache.hadoop.hive.ql.session.SessionState; @@ -92,8 +92,8 @@ private static final int WAIT_TIME = 3; private static final String[] PERF_KEYS = new String[] { - PerfLogger.PARSE, PerfLogger.COMPILE, PerfLogger.ANALYZE, PerfLogger.OPTIMIZER, - PerfLogger.GET_SPLITS, PerfLogger.RUN_TASKS, + PerfLoggerTokens.PARSE, PerfLoggerTokens.COMPILE, PerfLoggerTokens.ANALYZE, PerfLoggerTokens.OPTIMIZER, + PerfLoggerTokens.GET_SPLITS, PerfLoggerTokens.RUN_TASKS, }; private static void setupAtsExecutor(HiveConf conf) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveInputFormat.java b/ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveInputFormat.java index 9a7e9d9..39ebe9e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveInputFormat.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveInputFormat.java @@ -48,6 +48,7 @@ import org.apache.hadoop.hive.ql.exec.Operator; import org.apache.hadoop.hive.ql.exec.Utilities; import org.apache.hadoop.hive.ql.log.PerfLogger; +import org.apache.hadoop.hive.ql.log.PerfLoggerTokens; import org.apache.hadoop.hive.ql.parse.SplitSample; import org.apache.hadoop.hive.ql.plan.OperatorDesc; import org.apache.hadoop.hive.ql.plan.PartitionDesc; @@ -494,7 +495,7 @@ public int hashCode() { @Override public InputSplit[] getSplits(JobConf job, int numSplits) throws IOException { PerfLogger perfLogger = SessionState.getPerfLogger(); - perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.GET_SPLITS); + perfLogger.PerfLogBegin(CLASS_NAME, PerfLoggerTokens.GET_SPLITS); init(job); ArrayList result = new ArrayList(); @@ -522,7 +523,7 @@ public int hashCode() { } } catch (Exception e) { LOG.error("Error checking non-combinable path", e); - perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.GET_SPLITS); + perfLogger.PerfLogEnd(CLASS_NAME, PerfLoggerTokens.GET_SPLITS); throw new IOException(e); } } @@ -568,7 +569,7 @@ public int hashCode() { Utilities.clearWorkMapForConf(job); LOG.info("Number of all splits " + result.size()); - perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.GET_SPLITS); + perfLogger.PerfLogEnd(CLASS_NAME, PerfLoggerTokens.GET_SPLITS); return result.toArray(new InputSplit[result.size()]); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java b/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java index 010b88c..109568e 100755 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java @@ -51,6 +51,7 @@ import org.apache.hadoop.hive.ql.exec.Utilities; import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; import org.apache.hadoop.hive.ql.log.PerfLogger; +import org.apache.hadoop.hive.ql.log.PerfLoggerTokens; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc; import org.apache.hadoop.hive.ql.plan.MapWork; @@ -471,7 +472,7 @@ private void addSplitsForGroup(List dirs, TableScanOperator tableScan, Job public InputSplit[] getSplits(JobConf job, int numSplits) throws IOException { PerfLogger perfLogger = SessionState.getPerfLogger(); - perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.GET_SPLITS); + perfLogger.PerfLogBegin(CLASS_NAME, PerfLoggerTokens.GET_SPLITS); init(job); Path[] dirs = getInputPaths(job); JobConf newjob = new JobConf(job); @@ -568,7 +569,7 @@ private void addSplitsForGroup(List dirs, TableScanOperator tableScan, Job if (LOG.isInfoEnabled()) { LOG.info("number of splits " + result.size()); } - perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.GET_SPLITS); + perfLogger.PerfLogEnd(CLASS_NAME, PerfLoggerTokens.GET_SPLITS); return result.toArray(new HiveInputSplit[result.size()]); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/log/LogDivertAppender.java b/ql/src/java/org/apache/hadoop/hive/ql/log/LogDivertAppender.java index e697b54..8cb479d 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/log/LogDivertAppender.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/log/LogDivertAppender.java @@ -23,7 +23,7 @@ import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.Driver; import org.apache.hadoop.hive.ql.exec.Task; -import org.apache.hadoop.hive.ql.log.PerfLogger; +import org.apache.hadoop.hive.ql.log.PerfLoggerTokens; import org.apache.hadoop.hive.ql.session.OperationLog; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.core.LogEvent; @@ -84,8 +84,7 @@ /* Patterns that are included in performance logging level. * In performance mode, show execution and performance logger messages. */ - private static final Pattern performanceIncludeNamePattern = Pattern.compile( - executionIncludeNamePattern.pattern() + "|" + PerfLogger.class.getName()); + private static final Pattern performanceIncludeNamePattern = Pattern.compile(executionIncludeNamePattern.pattern() + "|" + PerfLoggerTokens.class.getName()); private void setCurrentNamePattern(OperationLog.LoggingLevel mode) { if (mode == OperationLog.LoggingLevel.VERBOSE) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java index 5b908e8..3f8d492 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java @@ -131,7 +131,7 @@ import org.apache.hadoop.hive.ql.index.HiveIndexHandler; import org.apache.hadoop.hive.ql.io.AcidUtils; import org.apache.hadoop.hive.common.log.InPlaceUpdate; -import org.apache.hadoop.hive.ql.log.PerfLogger; +import org.apache.hadoop.hive.ql.log.PerfLoggerTokens; import org.apache.hadoop.hive.ql.optimizer.listbucketingpruner.ListBucketingPrunerUtils; import org.apache.hadoop.hive.ql.plan.AddPartitionDesc; import org.apache.hadoop.hive.ql.plan.DropTableDesc; @@ -155,6 +155,7 @@ import com.google.common.collect.Maps; import com.google.common.collect.Sets; import com.google.common.util.concurrent.ThreadFactoryBuilder; +import org.apache.hadoop.hive.ql.log.PerfLogger; /** * This class has functions that implement meta data/DDL operations using calls diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMetaStoreChecker.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMetaStoreChecker.java index 4add836..07c6f52 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMetaStoreChecker.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMetaStoreChecker.java @@ -39,6 +39,7 @@ import org.apache.hadoop.hive.common.StringInternUtils; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.ql.log.PerfLogger; +import org.apache.hadoop.hive.ql.log.PerfLoggerTokens; import org.apache.hadoop.hive.ql.session.SessionState; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -223,9 +224,9 @@ void checkTable(String dbName, String tableName, } else { List loadedPartitions = new ArrayList<>(); PerfLogger perfLogger = SessionState.getPerfLogger(); - perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.PARTITION_RETRIEVING); + perfLogger.PerfLogBegin(CLASS_NAME, PerfLoggerTokens.PARTITION_RETRIEVING); loadedPartitions.addAll(hive.getAllPartitionsOf(table)); - perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.PARTITION_RETRIEVING); + perfLogger.PerfLogEnd(CLASS_NAME, PerfLoggerTokens.PARTITION_RETRIEVING); parts = new PartitionIterable(loadedPartitions); } } else { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Transform.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Transform.java index 9ce9478..c21f186 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Transform.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Transform.java @@ -19,6 +19,7 @@ package org.apache.hadoop.hive.ql.optimizer; import org.apache.hadoop.hive.ql.log.PerfLogger; +import org.apache.hadoop.hive.ql.log.PerfLoggerTokens; import org.apache.hadoop.hive.ql.parse.ParseContext; import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.session.SessionState; @@ -42,15 +43,15 @@ public void beginPerfLogging() { PerfLogger perfLogger = SessionState.getPerfLogger(); - perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.OPTIMIZER); + perfLogger.PerfLogBegin(this.getClass().getName(), PerfLoggerTokens.OPTIMIZER); } public void endPerfLogging() { PerfLogger perfLogger = SessionState.getPerfLogger(); - perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.OPTIMIZER); + perfLogger.PerfLogEnd(this.getClass().getName(), PerfLoggerTokens.OPTIMIZER); } public void endPerfLogging(String additionalInfo) { PerfLogger perfLogger = SessionState.getPerfLogger(); - perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.OPTIMIZER, additionalInfo); + perfLogger.PerfLogEnd(this.getClass().getName(), PerfLoggerTokens.OPTIMIZER, additionalInfo); } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java index 2acfef7..ddf36fc 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java @@ -38,6 +38,7 @@ import org.apache.hadoop.hive.ql.exec.FunctionRegistry; import org.apache.hadoop.hive.ql.exec.TableScanOperator; import org.apache.hadoop.hive.ql.log.PerfLogger; +import org.apache.hadoop.hive.ql.log.PerfLoggerTokens; import org.apache.hadoop.hive.ql.metadata.Hive; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.metadata.Partition; @@ -426,7 +427,7 @@ private static PrunedPartitionList getPartitionsFromServer(Table tab, boolean hasUnknownPartitions = false; PerfLogger perfLogger = SessionState.getPerfLogger(); if (!doEvalClientSide) { - perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.PARTITION_RETRIEVING); + perfLogger.PerfLogBegin(CLASS_NAME, PerfLoggerTokens.PARTITION_RETRIEVING); try { hasUnknownPartitions = Hive.get().getPartitionsByExpr( tab, compactExpr, conf, partitions); @@ -435,7 +436,7 @@ private static PrunedPartitionList getPartitionsFromServer(Table tab, LOG.warn("Metastore doesn't support getPartitionsByExpr", ime); doEvalClientSide = true; } finally { - perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.PARTITION_RETRIEVING); + perfLogger.PerfLogEnd(CLASS_NAME, PerfLoggerTokens.PARTITION_RETRIEVING); } } if (doEvalClientSide) { @@ -457,9 +458,9 @@ private static PrunedPartitionList getPartitionsFromServer(Table tab, private static Set getAllPartitions(Table tab) throws HiveException { PerfLogger perfLogger = SessionState.getPerfLogger(); - perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.PARTITION_RETRIEVING); + perfLogger.PerfLogBegin(CLASS_NAME, PerfLoggerTokens.PARTITION_RETRIEVING); Set result = Hive.get().getAllPartitionsOf(tab); - perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.PARTITION_RETRIEVING); + perfLogger.PerfLogEnd(CLASS_NAME, PerfLoggerTokens.PARTITION_RETRIEVING); return result; } @@ -475,7 +476,7 @@ private static PrunedPartitionList getPartitionsFromServer(Table tab, static private boolean pruneBySequentialScan(Table tab, List partitions, ExprNodeGenericFuncDesc prunerExpr, HiveConf conf) throws HiveException, MetaException { PerfLogger perfLogger = SessionState.getPerfLogger(); - perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.PRUNE_LISTING); + perfLogger.PerfLogBegin(CLASS_NAME, PerfLoggerTokens.PRUNE_LISTING); List partNames = Hive.get().getPartitionNames( tab.getDbName(), tab.getTableName(), (short) -1); @@ -486,13 +487,13 @@ static private boolean pruneBySequentialScan(Table tab, List partitio boolean hasUnknownPartitions = prunePartitionNames( partCols, partColTypeInfos, prunerExpr, defaultPartitionName, partNames); - perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.PRUNE_LISTING); + perfLogger.PerfLogEnd(CLASS_NAME, PerfLoggerTokens.PRUNE_LISTING); - perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.PARTITION_RETRIEVING); + perfLogger.PerfLogBegin(CLASS_NAME, PerfLoggerTokens.PARTITION_RETRIEVING); if (!partNames.isEmpty()) { partitions.addAll(Hive.get().getPartitionsByNames(tab, partNames)); } - perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.PARTITION_RETRIEVING); + perfLogger.PerfLogEnd(CLASS_NAME, PerfLoggerTokens.PARTITION_RETRIEVING); return hasUnknownPartitions; } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java index 5d640be..e39bf72 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java @@ -138,6 +138,7 @@ import org.apache.hadoop.hive.ql.exec.RowSchema; import org.apache.hadoop.hive.ql.lib.Node; import org.apache.hadoop.hive.ql.log.PerfLogger; +import org.apache.hadoop.hive.ql.log.PerfLoggerTokens; import org.apache.hadoop.hive.ql.metadata.Hive; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.metadata.Table; @@ -1343,7 +1344,7 @@ public RelNode apply(RelOptCluster cluster, RelOptSchema relOptSchema, SchemaPlu PerfLogger perfLogger = SessionState.getPerfLogger(); // 1. Gen Calcite Plan - perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.OPTIMIZER); + perfLogger.PerfLogBegin(this.getClass().getName(), PerfLoggerTokens.OPTIMIZER); try { calciteGenPlan = genLogicalPlan(getQB(), true, null, null); // if it is to create view, we do not use table alias @@ -1355,7 +1356,7 @@ public RelNode apply(RelOptCluster cluster, RelOptSchema relOptSchema, SchemaPlu semanticException = e; throw new RuntimeException(e); } - perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.OPTIMIZER, "Calcite: Plan generation"); + perfLogger.PerfLogEnd(this.getClass().getName(), PerfLoggerTokens.OPTIMIZER, "Calcite: Plan generation"); // Create executor RexExecutor executorProvider = new HiveRexExecutorImpl(optCluster); @@ -1390,7 +1391,7 @@ public RelNode apply(RelOptCluster cluster, RelOptSchema relOptSchema, SchemaPlu // If join optimizations failed because of missing stats, we continue with // the rest of optimizations if (profilesCBO.contains(ExtendedCBOProfile.JOIN_REORDERING)) { - perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.OPTIMIZER); + perfLogger.PerfLogBegin(this.getClass().getName(), PerfLoggerTokens.OPTIMIZER); try { List list = Lists.newArrayList(); list.add(mdProvider.getMetadataProvider()); @@ -1427,26 +1428,26 @@ public RelNode apply(RelOptCluster cluster, RelOptSchema relOptSchema, SchemaPlu throw e; } } - perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.OPTIMIZER, "Calcite: Join Reordering"); + perfLogger.PerfLogEnd(this.getClass().getName(), PerfLoggerTokens.OPTIMIZER, "Calcite: Join Reordering"); } else { calciteOptimizedPlan = calcitePreCboPlan; disableSemJoinReordering = false; } // 4. Run other optimizations that do not need stats - perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.OPTIMIZER); + perfLogger.PerfLogBegin(this.getClass().getName(), PerfLoggerTokens.OPTIMIZER); calciteOptimizedPlan = hepPlan(calciteOptimizedPlan, false, mdProvider.getMetadataProvider(), null, HepMatchOrder.BOTTOM_UP, ProjectRemoveRule.INSTANCE, UnionMergeRule.INSTANCE, HiveProjectMergeRule.INSTANCE_NO_FORCE, HiveAggregateProjectMergeRule.INSTANCE, HiveJoinCommuteRule.INSTANCE); - perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.OPTIMIZER, "Calcite: Optimizations without stats"); + perfLogger.PerfLogEnd(this.getClass().getName(), PerfLoggerTokens.OPTIMIZER, "Calcite: Optimizations without stats"); // 5. Materialized view based rewriting // We disable it for CTAS and MV creation queries (trying to avoid any problem // due to data freshness) if (conf.getBoolVar(ConfVars.HIVE_MATERIALIZED_VIEW_ENABLE_AUTO_REWRITING) && !getQB().isMaterializedView() && !getQB().isCTAS()) { - perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.OPTIMIZER); + perfLogger.PerfLogBegin(this.getClass().getName(), PerfLoggerTokens.OPTIMIZER); // Use Calcite cost model for view rewriting RelMetadataProvider calciteMdProvider = DefaultRelMetadataProvider.INSTANCE; RelMetadataQuery.THREAD_PROVIDERS.set(JaninoRelMetadataProvider.of(calciteMdProvider)); @@ -1494,14 +1495,14 @@ public RelOptMaterialization apply(RelOptMaterialization materialization) { } // Restore default cost model RelMetadataQuery.THREAD_PROVIDERS.set(JaninoRelMetadataProvider.of(mdProvider.getMetadataProvider())); - perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.OPTIMIZER, "Calcite: View-based rewriting"); + perfLogger.PerfLogEnd(this.getClass().getName(), PerfLoggerTokens.OPTIMIZER, "Calcite: View-based rewriting"); } // 6. Run aggregate-join transpose (cost based) // If it failed because of missing stats, we continue with // the rest of optimizations if (conf.getBoolVar(ConfVars.AGGR_JOIN_TRANSPOSE)) { - perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.OPTIMIZER); + perfLogger.PerfLogBegin(this.getClass().getName(), PerfLoggerTokens.OPTIMIZER); try { calciteOptimizedPlan = hepPlan(calciteOptimizedPlan, false, mdProvider.getMetadataProvider(), null, HepMatchOrder.BOTTOM_UP, HiveAggregateJoinTransposeRule.INSTANCE); @@ -1514,29 +1515,29 @@ public RelOptMaterialization apply(RelOptMaterialization materialization) { throw e; } } - perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.OPTIMIZER, "Calcite: Aggregate join transpose"); + perfLogger.PerfLogEnd(this.getClass().getName(), PerfLoggerTokens.OPTIMIZER, "Calcite: Aggregate join transpose"); } // 7.convert Join + GBy to semijoin // run this rule at later stages, since many calcite rules cant deal with semijoin if (conf.getBoolVar(ConfVars.SEMIJOIN_CONVERSION)) { - perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.OPTIMIZER); + perfLogger.PerfLogBegin(this.getClass().getName(), PerfLoggerTokens.OPTIMIZER); calciteOptimizedPlan = hepPlan(calciteOptimizedPlan, false, mdProvider.getMetadataProvider(), null, HiveSemiJoinRule.INSTANCE); - perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.OPTIMIZER, "Calcite: Semijoin conversion"); + perfLogger.PerfLogEnd(this.getClass().getName(), PerfLoggerTokens.OPTIMIZER, "Calcite: Semijoin conversion"); } // 8. Run rule to fix windowing issue when it is done over // aggregation columns (HIVE-10627) if (profilesCBO.contains(ExtendedCBOProfile.WINDOWING_POSTPROCESSING)) { - perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.OPTIMIZER); + perfLogger.PerfLogBegin(this.getClass().getName(), PerfLoggerTokens.OPTIMIZER); calciteOptimizedPlan = hepPlan(calciteOptimizedPlan, false, mdProvider.getMetadataProvider(), null, HepMatchOrder.BOTTOM_UP, HiveWindowingFixRule.INSTANCE); - perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.OPTIMIZER, "Calcite: Window fixing rule"); + perfLogger.PerfLogEnd(this.getClass().getName(), PerfLoggerTokens.OPTIMIZER, "Calcite: Window fixing rule"); } // 9. Apply Druid transformation rules - perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.OPTIMIZER); + perfLogger.PerfLogBegin(this.getClass().getName(), PerfLoggerTokens.OPTIMIZER); calciteOptimizedPlan = hepPlan(calciteOptimizedPlan, false, mdProvider.getMetadataProvider(), null, HepMatchOrder.BOTTOM_UP, DruidRules.FILTER, @@ -1551,11 +1552,11 @@ public RelOptMaterialization apply(RelOptMaterialization materialization) { DruidRules.SORT, DruidRules.PROJECT_SORT_TRANSPOSE ); - perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.OPTIMIZER, "Calcite: Druid transformation rules"); + perfLogger.PerfLogEnd(this.getClass().getName(), PerfLoggerTokens.OPTIMIZER, "Calcite: Druid transformation rules"); // 10. Run rules to aid in translation from Calcite tree to Hive tree if (HiveConf.getBoolVar(conf, ConfVars.HIVE_CBO_RETPATH_HIVEOP)) { - perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.OPTIMIZER); + perfLogger.PerfLogBegin(this.getClass().getName(), PerfLoggerTokens.OPTIMIZER); // 10.1. Merge join into multijoin operators (if possible) calciteOptimizedPlan = hepPlan(calciteOptimizedPlan, true, mdProvider.getMetadataProvider(), null, HepMatchOrder.BOTTOM_UP, HiveJoinProjectTransposeRule.BOTH_PROJECT_INCLUDE_OUTER, @@ -1578,7 +1579,7 @@ public RelOptMaterialization apply(RelOptMaterialization materialization) { calciteOptimizedPlan = hepPlan(calciteOptimizedPlan, false, mdProvider.getMetadataProvider(), null, HepMatchOrder.BOTTOM_UP, HiveInsertExchange4JoinRule.EXCHANGE_BELOW_JOIN, HiveInsertExchange4JoinRule.EXCHANGE_BELOW_MULTIJOIN); - perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.OPTIMIZER, "Calcite: Translation from Calcite tree to Hive tree"); + perfLogger.PerfLogEnd(this.getClass().getName(), PerfLoggerTokens.OPTIMIZER, "Calcite: Translation from Calcite tree to Hive tree"); } if (LOG.isDebugEnabled() && !conf.getBoolVar(ConfVars.HIVE_IN_TEST)) { @@ -1615,34 +1616,34 @@ private RelNode applyPreJoinOrderingTransforms(RelNode basePlan, RelMetadataProv final int minNumORClauses = conf.getIntVar(HiveConf.ConfVars.HIVEPOINTLOOKUPOPTIMIZERMIN); //0. SetOp rewrite - perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.OPTIMIZER); + perfLogger.PerfLogBegin(this.getClass().getName(), PerfLoggerTokens.OPTIMIZER); basePlan = hepPlan(basePlan, true, mdProvider, null, HepMatchOrder.BOTTOM_UP, HiveProjectOverIntersectRemoveRule.INSTANCE, HiveIntersectMergeRule.INSTANCE); - perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.OPTIMIZER, + perfLogger.PerfLogEnd(this.getClass().getName(), PerfLoggerTokens.OPTIMIZER, "Calcite: HiveProjectOverIntersectRemoveRule and HiveIntersectMerge rules"); - perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.OPTIMIZER); + perfLogger.PerfLogBegin(this.getClass().getName(), PerfLoggerTokens.OPTIMIZER); basePlan = hepPlan(basePlan, false, mdProvider, executorProvider, HepMatchOrder.BOTTOM_UP, HiveIntersectRewriteRule.INSTANCE); - perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.OPTIMIZER, + perfLogger.PerfLogEnd(this.getClass().getName(), PerfLoggerTokens.OPTIMIZER, "Calcite: HiveIntersectRewrite rule"); - perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.OPTIMIZER); + perfLogger.PerfLogBegin(this.getClass().getName(), PerfLoggerTokens.OPTIMIZER); basePlan = hepPlan(basePlan, false, mdProvider, executorProvider, HepMatchOrder.BOTTOM_UP, HiveExceptRewriteRule.INSTANCE); - perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.OPTIMIZER, + perfLogger.PerfLogEnd(this.getClass().getName(), PerfLoggerTokens.OPTIMIZER, "Calcite: HiveExceptRewrite rule"); //1. Distinct aggregate rewrite // Run this optimization early, since it is expanding the operator pipeline. if (!conf.getVar(HiveConf.ConfVars.HIVE_EXECUTION_ENGINE).equals("mr") && conf.getBoolVar(HiveConf.ConfVars.HIVEOPTIMIZEDISTINCTREWRITE)) { - perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.OPTIMIZER); + perfLogger.PerfLogBegin(this.getClass().getName(), PerfLoggerTokens.OPTIMIZER); // Its not clear, if this rewrite is always performant on MR, since extra map phase // introduced for 2nd MR job may offset gains of this multi-stage aggregation. // We need a cost model for MR to enable this on MR. basePlan = hepPlan(basePlan, true, mdProvider, executorProvider, HiveExpandDistinctAggregatesRule.INSTANCE); - perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.OPTIMIZER, + perfLogger.PerfLogEnd(this.getClass().getName(), PerfLoggerTokens.OPTIMIZER, "Calcite: Prejoin ordering transformation, Distinct aggregate rewrite"); } @@ -1651,10 +1652,10 @@ private RelNode applyPreJoinOrderingTransforms(RelNode basePlan, RelMetadataProv // add on-clauses for old style Join Syntax // Ex: select * from R1 join R2 where ((R1.x=R2.x) and R1.y<10) or // ((R1.x=R2.x) and R1.z=10)) and rand(1) < 0.1 - perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.OPTIMIZER); + perfLogger.PerfLogBegin(this.getClass().getName(), PerfLoggerTokens.OPTIMIZER); basePlan = hepPlan(basePlan, false, mdProvider, executorProvider, HepMatchOrder.ARBITRARY, new HivePreFilteringRule(maxCNFNodeCount)); - perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.OPTIMIZER, + perfLogger.PerfLogEnd(this.getClass().getName(), PerfLoggerTokens.OPTIMIZER, "Calcite: Prejoin ordering transformation, factor out common filter elements and separating deterministic vs non-deterministic UDF"); // 3. Run exhaustive PPD, add not null filters, transitive inference, @@ -1690,10 +1691,10 @@ private RelNode applyPreJoinOrderingTransforms(RelNode basePlan, RelMetadataProv rules.add(HiveSortLimitPullUpConstantsRule.INSTANCE); rules.add(HiveUnionPullUpConstantsRule.INSTANCE); rules.add(HiveAggregatePullUpConstantsRule.INSTANCE); - perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.OPTIMIZER); + perfLogger.PerfLogBegin(this.getClass().getName(), PerfLoggerTokens.OPTIMIZER); basePlan = hepPlan(basePlan, true, mdProvider, executorProvider, HepMatchOrder.BOTTOM_UP, rules.toArray(new RelOptRule[rules.size()])); - perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.OPTIMIZER, + perfLogger.PerfLogEnd(this.getClass().getName(), PerfLoggerTokens.OPTIMIZER, "Calcite: Prejoin ordering transformation, PPD, not null predicates, transitive inference, constant folding"); // 4. Push down limit through outer join @@ -1701,7 +1702,7 @@ private RelNode applyPreJoinOrderingTransforms(RelNode basePlan, RelMetadataProv // Ex: select * from R1 left outer join R2 where ((R1.x=R2.x) and R1.y<10) or // ((R1.x=R2.x) and R1.z=10)) and rand(1) < 0.1 order by R1.x limit 10 if (conf.getBoolVar(HiveConf.ConfVars.HIVE_OPTIMIZE_LIMIT_TRANSPOSE)) { - perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.OPTIMIZER); + perfLogger.PerfLogBegin(this.getClass().getName(), PerfLoggerTokens.OPTIMIZER); // This should be a cost based decision, but till we enable the extended cost // model, we will use the given value for the variable final float reductionProportion = HiveConf.getFloatVar(conf, @@ -1714,48 +1715,48 @@ private RelNode applyPreJoinOrderingTransforms(RelNode basePlan, RelMetadataProv basePlan = hepPlan(basePlan, true, mdProvider, executorProvider, HepMatchOrder.BOTTOM_UP, new HiveSortRemoveRule(reductionProportion, reductionTuples), HiveProjectSortTransposeRule.INSTANCE); - perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.OPTIMIZER, + perfLogger.PerfLogEnd(this.getClass().getName(), PerfLoggerTokens.OPTIMIZER, "Calcite: Prejoin ordering transformation, Push down limit through outer join"); } // 5. Push Down Semi Joins - perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.OPTIMIZER); + perfLogger.PerfLogBegin(this.getClass().getName(), PerfLoggerTokens.OPTIMIZER); basePlan = hepPlan(basePlan, true, mdProvider, executorProvider, SemiJoinJoinTransposeRule.INSTANCE, SemiJoinFilterTransposeRule.INSTANCE, SemiJoinProjectTransposeRule.INSTANCE); - perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.OPTIMIZER, + perfLogger.PerfLogEnd(this.getClass().getName(), PerfLoggerTokens.OPTIMIZER, "Calcite: Prejoin ordering transformation, Push Down Semi Joins"); // 6. Apply Partition Pruning - perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.OPTIMIZER); + perfLogger.PerfLogBegin(this.getClass().getName(), PerfLoggerTokens.OPTIMIZER); basePlan = hepPlan(basePlan, false, mdProvider, executorProvider, new HivePartitionPruneRule(conf)); - perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.OPTIMIZER, + perfLogger.PerfLogEnd(this.getClass().getName(), PerfLoggerTokens.OPTIMIZER, "Calcite: Prejoin ordering transformation, Partition Pruning"); // 7. Projection Pruning (this introduces select above TS & hence needs to be run last due to PP) - perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.OPTIMIZER); + perfLogger.PerfLogBegin(this.getClass().getName(), PerfLoggerTokens.OPTIMIZER); HiveRelFieldTrimmer fieldTrimmer = new HiveRelFieldTrimmer(null, HiveRelFactories.HIVE_BUILDER.create(cluster, null), profilesCBO.contains(ExtendedCBOProfile.JOIN_REORDERING)); basePlan = fieldTrimmer.trim(basePlan); - perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.OPTIMIZER, + perfLogger.PerfLogEnd(this.getClass().getName(), PerfLoggerTokens.OPTIMIZER, "Calcite: Prejoin ordering transformation, Projection Pruning"); // 8. Merge, remove and reduce Project if possible - perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.OPTIMIZER); + perfLogger.PerfLogBegin(this.getClass().getName(), PerfLoggerTokens.OPTIMIZER); basePlan = hepPlan(basePlan, false, mdProvider, executorProvider, HiveProjectMergeRule.INSTANCE, ProjectRemoveRule.INSTANCE); - perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.OPTIMIZER, + perfLogger.PerfLogEnd(this.getClass().getName(), PerfLoggerTokens.OPTIMIZER, "Calcite: Prejoin ordering transformation, Merge Project-Project"); // 9. Rerun PPD through Project as column pruning would have introduced // DT above scans; By pushing filter just above TS, Hive can push it into // storage (incase there are filters on non partition cols). This only // matches FIL-PROJ-TS - perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.OPTIMIZER); + perfLogger.PerfLogBegin(this.getClass().getName(), PerfLoggerTokens.OPTIMIZER); basePlan = hepPlan(basePlan, true, mdProvider, executorProvider, HiveFilterProjectTSTransposeRule.INSTANCE, HiveFilterProjectTSTransposeRule.INSTANCE_DRUID, HiveProjectFilterPullUpConstantsRule.INSTANCE); - perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.OPTIMIZER, + perfLogger.PerfLogEnd(this.getClass().getName(), PerfLoggerTokens.OPTIMIZER, "Calcite: Prejoin ordering transformation, Rerun PPD"); return basePlan; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java index 078704e..a38fb12 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java @@ -38,6 +38,7 @@ import org.apache.hadoop.hive.ql.hooks.ReadEntity; import org.apache.hadoop.hive.ql.hooks.WriteEntity; import org.apache.hadoop.hive.ql.log.PerfLogger; +import org.apache.hadoop.hive.ql.log.PerfLoggerTokens; import org.apache.hadoop.hive.ql.metadata.Hive; import org.apache.hadoop.hive.ql.optimizer.ConstantPropagate; import org.apache.hadoop.hive.ql.optimizer.ConstantPropagateProcCtx.ConstantPropagateOption; @@ -92,49 +93,49 @@ protected void optimizeOperatorPlan(ParseContext pCtx, Set inputs, PerfLogger perfLogger = SessionState.getPerfLogger(); // Create the context for the walker OptimizeTezProcContext procCtx = new OptimizeTezProcContext(conf, pCtx, inputs, outputs); - perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.TEZ_COMPILER); + perfLogger.PerfLogBegin(this.getClass().getName(), PerfLoggerTokens.TEZ_COMPILER); // setup dynamic partition pruning where possible runDynamicPartitionPruning(procCtx, inputs, outputs); - perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.TEZ_COMPILER, "Setup dynamic partition pruning"); + perfLogger.PerfLogEnd(this.getClass().getName(), PerfLoggerTokens.TEZ_COMPILER, "Setup dynamic partition pruning"); - perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.TEZ_COMPILER); + perfLogger.PerfLogBegin(this.getClass().getName(), PerfLoggerTokens.TEZ_COMPILER); // setup stats in the operator plan runStatsAnnotation(procCtx); - perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.TEZ_COMPILER, "Setup stats in the operator plan"); + perfLogger.PerfLogEnd(this.getClass().getName(), PerfLoggerTokens.TEZ_COMPILER, "Setup stats in the operator plan"); - perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.TEZ_COMPILER); + perfLogger.PerfLogBegin(this.getClass().getName(), PerfLoggerTokens.TEZ_COMPILER); // run the optimizations that use stats for optimization runStatsDependentOptimizations(procCtx, inputs, outputs); - perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.TEZ_COMPILER, "Run the optimizations that use stats for optimization"); + perfLogger.PerfLogEnd(this.getClass().getName(), PerfLoggerTokens.TEZ_COMPILER, "Run the optimizations that use stats for optimization"); // Removing semijoin optimization when it may not be beneficial removeSemijoinOptimizationByBenefit(procCtx); - perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.TEZ_COMPILER); + perfLogger.PerfLogBegin(this.getClass().getName(), PerfLoggerTokens.TEZ_COMPILER); // Remove any parallel edge between semijoin and mapjoin. removeSemijoinsParallelToMapJoin(procCtx); - perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.TEZ_COMPILER, "Run the optimizations that use stats for optimization"); + perfLogger.PerfLogEnd(this.getClass().getName(), PerfLoggerTokens.TEZ_COMPILER, "Run the optimizations that use stats for optimization"); - perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.TEZ_COMPILER); + perfLogger.PerfLogBegin(this.getClass().getName(), PerfLoggerTokens.TEZ_COMPILER); // Remove semijoin optimization if it creates a cycle with mapside joins removeSemiJoinCyclesDueToMapsideJoins(procCtx); - perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.TEZ_COMPILER, "Remove semijoin optimizations if it creates a cycle with mapside join"); + perfLogger.PerfLogEnd(this.getClass().getName(), PerfLoggerTokens.TEZ_COMPILER, "Remove semijoin optimizations if it creates a cycle with mapside join"); - perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.TEZ_COMPILER); + perfLogger.PerfLogBegin(this.getClass().getName(), PerfLoggerTokens.TEZ_COMPILER); // Remove semijoin optimization if SMB join is created. removeSemijoinOptimizationFromSMBJoins(procCtx); - perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.TEZ_COMPILER, "Remove semijoin optimizations if needed"); + perfLogger.PerfLogEnd(this.getClass().getName(), PerfLoggerTokens.TEZ_COMPILER, "Remove semijoin optimizations if needed"); - perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.TEZ_COMPILER); + perfLogger.PerfLogBegin(this.getClass().getName(), PerfLoggerTokens.TEZ_COMPILER); // Remove bloomfilter if no stats generated removeSemiJoinIfNoStats(procCtx); - perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.TEZ_COMPILER, "Remove bloom filter optimizations if needed"); + perfLogger.PerfLogEnd(this.getClass().getName(), PerfLoggerTokens.TEZ_COMPILER, "Remove bloom filter optimizations if needed"); - perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.TEZ_COMPILER); + perfLogger.PerfLogBegin(this.getClass().getName(), PerfLoggerTokens.TEZ_COMPILER); // after the stats phase we might have some cyclic dependencies that we need // to take care of. runCycleAnalysisForPartitionPruning(procCtx, inputs, outputs); - perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.TEZ_COMPILER, "Run cycle analysis for partition pruning"); + perfLogger.PerfLogEnd(this.getClass().getName(), PerfLoggerTokens.TEZ_COMPILER, "Run cycle analysis for partition pruning"); // need a new run of the constant folding because we might have created lots // of "and true and true" conditions. @@ -405,8 +406,8 @@ protected void generateTaskTree(List> rootTasks, Pa List> mvTask, Set inputs, Set outputs) throws SemanticException { - PerfLogger perfLogger = SessionState.getPerfLogger(); - perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.TEZ_COMPILER); + PerfLogger perfLogger = SessionState.getPerfLogger(); + perfLogger.PerfLogBegin(this.getClass().getName(), PerfLoggerTokens.TEZ_COMPILER); ParseContext tempParseContext = getParseContext(pCtx, rootTasks); GenTezUtils utils = new GenTezUtils(); GenTezWork genTezWork = new GenTezWork(utils); @@ -490,7 +491,7 @@ protected void generateTaskTree(List> rootTasks, Pa LOG.debug("Handling AppMasterEventOperator: " + event); GenTezUtils.processAppMasterEvent(procCtx, event); } - perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.TEZ_COMPILER, "generateTaskTree"); + perfLogger.PerfLogEnd(this.getClass().getName(), PerfLoggerTokens.TEZ_COMPILER, "generateTaskTree"); } @Override @@ -552,7 +553,7 @@ protected void decideExecMode(List> rootTasks, Cont protected void optimizeTaskPlan(List> rootTasks, ParseContext pCtx, Context ctx) throws SemanticException { PerfLogger perfLogger = SessionState.getPerfLogger(); - perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.TEZ_COMPILER); + perfLogger.PerfLogBegin(this.getClass().getName(), PerfLoggerTokens.TEZ_COMPILER); PhysicalContext physicalCtx = new PhysicalContext(conf, pCtx, pCtx.getContext(), rootTasks, pCtx.getFetchTask()); @@ -615,7 +616,7 @@ protected void optimizeTaskPlan(List> rootTasks, Pa new AnnotateRunTimeStatsOptimizer().resolve(physicalCtx); } - perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.TEZ_COMPILER, "optimizeTaskPlan"); + perfLogger.PerfLogEnd(this.getClass().getName(), PerfLoggerTokens.TEZ_COMPILER, "optimizeTaskPlan"); return; } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkCompiler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkCompiler.java index 682b987..cd04abe 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkCompiler.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkCompiler.java @@ -61,6 +61,7 @@ import org.apache.hadoop.hive.ql.lib.RuleRegExp; import org.apache.hadoop.hive.ql.lib.TypeRule; import org.apache.hadoop.hive.ql.log.PerfLogger; +import org.apache.hadoop.hive.ql.log.PerfLoggerTokens; import org.apache.hadoop.hive.ql.optimizer.ConstantPropagate; import org.apache.hadoop.hive.ql.optimizer.DynamicPartitionPruningOptimization; import org.apache.hadoop.hive.ql.optimizer.SparkRemoveDynamicPruningBySize; @@ -108,7 +109,7 @@ public SparkCompiler() { @Override protected void optimizeOperatorPlan(ParseContext pCtx, Set inputs, Set outputs) throws SemanticException { - PERF_LOGGER.PerfLogBegin(CLASS_NAME, PerfLogger.SPARK_OPTIMIZE_OPERATOR_TREE); + PERF_LOGGER.PerfLogBegin(CLASS_NAME, PerfLoggerTokens.SPARK_OPTIMIZE_OPERATOR_TREE); OptimizeSparkProcContext procCtx = new OptimizeSparkProcContext(conf, pCtx, inputs, outputs); @@ -127,7 +128,7 @@ protected void optimizeOperatorPlan(ParseContext pCtx, Set inputs, // Remove cyclic dependencies for DPP runCycleAnalysisForPartitionPruning(procCtx); - PERF_LOGGER.PerfLogEnd(CLASS_NAME, PerfLogger.SPARK_OPTIMIZE_OPERATOR_TREE); + PERF_LOGGER.PerfLogEnd(CLASS_NAME, PerfLoggerTokens.SPARK_OPTIMIZE_OPERATOR_TREE); } private void runCycleAnalysisForPartitionPruning(OptimizeSparkProcContext procCtx) { @@ -318,7 +319,7 @@ private void runJoinOptimizations(OptimizeSparkProcContext procCtx) throws Seman protected void generateTaskTree(List> rootTasks, ParseContext pCtx, List> mvTask, Set inputs, Set outputs) throws SemanticException { - PERF_LOGGER.PerfLogBegin(CLASS_NAME, PerfLogger.SPARK_GENERATE_TASK_TREE); + PERF_LOGGER.PerfLogBegin(CLASS_NAME, PerfLoggerTokens.SPARK_GENERATE_TASK_TREE); GenSparkUtils utils = GenSparkUtils.getUtils(); utils.resetSequenceNumber(); @@ -387,7 +388,7 @@ protected void generateTaskTree(List> rootTasks, Pa utils.processPartitionPruningSink(procCtx, (SparkPartitionPruningSinkOperator) prunerSink); } - PERF_LOGGER.PerfLogEnd(CLASS_NAME, PerfLogger.SPARK_GENERATE_TASK_TREE); + PERF_LOGGER.PerfLogEnd(CLASS_NAME, PerfLoggerTokens.SPARK_GENERATE_TASK_TREE); } private void generateTaskTreeHelper(GenSparkProcContext procCtx, List topNodes) @@ -530,7 +531,7 @@ protected void decideExecMode(List> rootTasks, Cont @Override protected void optimizeTaskPlan(List> rootTasks, ParseContext pCtx, Context ctx) throws SemanticException { - PERF_LOGGER.PerfLogBegin(CLASS_NAME, PerfLogger.SPARK_OPTIMIZE_TASK_TREE); + PERF_LOGGER.PerfLogBegin(CLASS_NAME, PerfLoggerTokens.SPARK_OPTIMIZE_TASK_TREE); PhysicalContext physicalCtx = new PhysicalContext(conf, pCtx, pCtx.getContext(), rootTasks, pCtx.getFetchTask()); @@ -581,7 +582,7 @@ protected void optimizeTaskPlan(List> rootTasks, Pa new AnnotateRunTimeStatsOptimizer().resolve(physicalCtx); } - PERF_LOGGER.PerfLogEnd(CLASS_NAME, PerfLogger.SPARK_OPTIMIZE_TASK_TREE); + PERF_LOGGER.PerfLogEnd(CLASS_NAME, PerfLoggerTokens.SPARK_OPTIMIZE_TASK_TREE); return; } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java index 7692512..50434be 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java @@ -69,7 +69,7 @@ import org.apache.hadoop.hive.ql.lockmgr.HiveTxnManager; import org.apache.hadoop.hive.ql.lockmgr.LockException; import org.apache.hadoop.hive.ql.lockmgr.TxnManagerFactory; -import org.apache.hadoop.hive.ql.log.PerfLogger; +import org.apache.hadoop.hive.ql.log.PerfLoggerTokens; import org.apache.hadoop.hive.ql.metadata.Hive; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.metadata.HiveUtils; @@ -93,6 +93,8 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.google.common.collect.Maps; +import org.apache.hadoop.hive.ql.log.PerfLogger; +import org.apache.hadoop.hive.ql.log.PerfLoggerFactory; /** * SessionState encapsulates common data associated with a session. @@ -1645,8 +1647,8 @@ public boolean isAuthorizationModeV2(){ /** * @return Tries to return an instance of the class whose name is configured in - * hive.exec.perf.logger, but if it can't it just returns an instance of - * the base PerfLogger class + hive.exec.perf.logger, but if it can't it just returns an instance of + the base PerfLoggerTokens class * */ public static PerfLogger getPerfLogger() { @@ -1656,16 +1658,16 @@ public static PerfLogger getPerfLogger() { /** * @param resetPerfLogger * @return Tries to return an instance of the class whose name is configured in - * hive.exec.perf.logger, but if it can't it just returns an instance of - * the base PerfLogger class + hive.exec.perf.logger, but if it can't it just returns an instance of + the base PerfLoggerTokens class * */ public static PerfLogger getPerfLogger(boolean resetPerfLogger) { SessionState ss = get(); if (ss == null) { - return PerfLogger.getPerfLogger(null, resetPerfLogger); + return PerfLoggerFactory.getPerfLogger(null, resetPerfLogger); } else { - return PerfLogger.getPerfLogger(ss.getConf(), resetPerfLogger); + return PerfLoggerFactory.getPerfLogger(ss.getConf(), resetPerfLogger); } } diff --git a/service/src/java/org/apache/hive/service/cli/operation/SQLOperation.java b/service/src/java/org/apache/hive/service/cli/operation/SQLOperation.java index 0b51591..ada8bf9 100644 --- a/service/src/java/org/apache/hive/service/cli/operation/SQLOperation.java +++ b/service/src/java/org/apache/hive/service/cli/operation/SQLOperation.java @@ -57,6 +57,7 @@ import org.apache.hadoop.hive.ql.exec.FetchTask; import org.apache.hadoop.hive.ql.exec.Task; import org.apache.hadoop.hive.ql.log.PerfLogger; +import org.apache.hadoop.hive.ql.log.PerfLoggerFactory; import org.apache.hadoop.hive.ql.metadata.Hive; import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse; import org.apache.hadoop.hive.ql.session.SessionState; @@ -334,7 +335,7 @@ public Object run() throws HiveSQLException { Hive.set(parentHive); // TODO: can this result in cross-thread reuse of session state? SessionState.setCurrentSessionState(parentSessionState); - PerfLogger.setPerfLogger(parentPerfLogger); + PerfLoggerFactory.setPerfLogger(parentPerfLogger); LogUtils.registerLoggingContext(queryState.getConf()); try { if (asyncPrepare) {