Index: conf/hive-default.xml =================================================================== --- conf/hive-default.xml (revision 1155439) +++ conf/hive-default.xml (working copy) @@ -1138,4 +1138,10 @@ by record readers + + hive.exec.perf.logger + org.apache.hadoop.hive.ql.log.PerfLogger + The class responsible logging client side performance metrics. Must be a subclass of org.apache.hadoop.hive.ql.log.PerfLogger + + Index: common/src/java/org/apache/hadoop/hive/conf/HiveConf.java =================================================================== --- common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (revision 1155439) +++ common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (working copy) @@ -457,6 +457,10 @@ HIVE_MAPPER_CANNOT_SPAN_MULTIPLE_PARTITIONS("hive.mapper.cannot.span.multiple.partitions", false), HIVE_REWORK_MAPREDWORK("hive.rework.mapredwork", false), HIVE_CONCATENATE_CHECK_INDEX ("hive.exec.concatenate.check.index", true), + + // The class responsible for logging client side performance metrics + // Must be a subclass of org.apache.hadoop.hive.ql.log.PerfLogger + HIVE_PERF_LOGGER("hive.exec.perf.logger", "org.apache.hadoop.hive.ql.log.PerfLogger"), ; public final String varname; Index: ql/src/java/org/apache/hadoop/hive/ql/log/PerfLogger.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/log/PerfLogger.java (revision 0) +++ ql/src/java/org/apache/hadoop/hive/ql/log/PerfLogger.java (revision 0) @@ -0,0 +1,123 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.log; + +import java.util.HashMap; +import java.util.Map; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConf.ConfVars; +import org.apache.hadoop.hive.ql.session.SessionState; +import org.apache.hadoop.util.ReflectionUtils; + +/** + * PerfLogger. + * + * Can be used to measure and log the time spent by a piece of code. + */ +public class PerfLogger { + public static final String ACQUIRE_READ_WRITE_LOCKS = "acquireReadWriteLocks"; + public static final String COMPILE = "compile"; + public static final String DO_AUTHORIZATION = "doAuthorization"; + public static final String DRIVER_EXECUTE = "Driver.execute"; + public static final String RELEASE_LOCKS = "releaseLocks"; + public static final String PRUNE_LISTING = "prune-listing"; + public static final String PARTITION_RETRIEVING = "partition-retrieving"; + public static final String PRE_HOOK = "PreHook."; + public static final String POST_HOOK = "PostHook."; + + protected static final ThreadLocal perfLogger = new ThreadLocal(); + + protected final Map perfKeyMaps = new HashMap(); + + static final private Log LOG = LogFactory.getLog(PerfLogger.class.getName()); + + protected PerfLogger() { + // Use getPerfLogger to get an instance of PerfLogger + } + + public static PerfLogger getPerfLogger() { + return getPerfLogger(false); + } + + /** + * Call this function to get an instance of PerfLogger. + * + * Use resetPerfLogger to require a new instance. Useful at the beginning of execution. + * + * @return Tries to return an instance of the class whose name is configured in + * hive.exec.perf.logger, but if it can't it just returns an instance of + * the base PerfLogger class + */ + public static PerfLogger getPerfLogger(boolean resetPerfLogger) { + if (perfLogger.get() == null || resetPerfLogger) { + if (SessionState.get() == null) { + perfLogger.set(new PerfLogger()); + } else { + HiveConf conf = SessionState.get().getConf(); + try { + perfLogger.set((PerfLogger) ReflectionUtils.newInstance(conf.getClassByName( + conf.getVar(ConfVars.HIVE_PERF_LOGGER)), conf)); + } catch (ClassNotFoundException e) { + LOG.error("Performance Logger Class not found:" + e.getMessage()); + perfLogger.set(new PerfLogger()); + } + } + } + return perfLogger.get(); + } + + /** + * Call this function when you start to measure time spent by a piece of code. + * @param _log the logging object to be used. + * @param method method or ID that identifies this perf log element. + */ + public void PerfLogBegin(Log _log, String method) { + long startTime = System.currentTimeMillis(); + _log.info(""); + perfKeyMaps.put(method, new Long(startTime)); + } + + /** + * Call this function in correspondence of PerfLogBegin to mark the end of the measurement. + * @param _log + * @param method + * @return long duration the difference between now and startTime, or -1 if startTime is null + */ + public long PerfLogEnd(Log _log, String method) { + Long startTime = perfKeyMaps.get(method); + long endTime = System.currentTimeMillis(); + long duration = -1; + StringBuilder sb = new StringBuilder(""); + _log.info(sb); + + return duration; + } +} Index: ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java (revision 1155439) +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java (working copy) @@ -44,6 +44,7 @@ import org.apache.hadoop.hive.ql.lib.NodeProcessor; import org.apache.hadoop.hive.ql.lib.Rule; import org.apache.hadoop.hive.ql.lib.RuleRegExp; +import org.apache.hadoop.hive.ql.log.PerfLogger; import org.apache.hadoop.hive.ql.metadata.Hive; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.metadata.Partition; @@ -320,8 +321,10 @@ List trueNames = null; List unknNames = null; - Utilities.PerfLogBegin(LOG, "prune-listing"); + PerfLogger perfLogger = PerfLogger.getPerfLogger(); + perfLogger.PerfLogBegin(LOG, PerfLogger.PRUNE_LISTING); + List partNames = Hive.get().getPartitionNames(tab.getDbName(), tab.getTableName(), (short) -1); @@ -366,9 +369,9 @@ LOG.debug("retained partition: " + partName); } } - Utilities.PerfLogEnd(LOG, "prune-listing"); + perfLogger.PerfLogEnd(LOG, PerfLogger.PRUNE_LISTING); - Utilities.PerfLogBegin(LOG, "partition-retrieving"); + perfLogger.PerfLogBegin(LOG, PerfLogger.PARTITION_RETRIEVING); if (trueNames != null) { List parts = Hive.get().getPartitionsByNames(tab, trueNames); true_parts.addAll(parts); @@ -377,7 +380,7 @@ List parts = Hive.get().getPartitionsByNames(tab, unknNames); unkn_parts.addAll(parts); } - Utilities.PerfLogEnd(LOG, "partition-retrieving"); + perfLogger.PerfLogEnd(LOG, PerfLogger.PARTITION_RETRIEVING); } /** Index: ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java (revision 1155439) +++ ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java (working copy) @@ -2004,43 +2004,7 @@ return false; } - private static ThreadLocal> perfKeyMaps = new ThreadLocal>(); - /** - * Call this function when you start to measure time spent by a piece of code. - * @param _log the logging object to be used. - * @param method method or ID that identifies this perf log element. - */ - public static void PerfLogBegin(Log _log, String method) { - long startTime = System.currentTimeMillis(); - _log.info(""); - if (perfKeyMaps.get() == null) { - perfKeyMaps.set(new HashMap()); - } - perfKeyMaps.get().put(method, new Long(startTime)); - } - - /** - * Call this function in correspondence of PerfLogBegin to mark the end of the measurement. - * @param _log - * @param method - */ - public static void PerfLogEnd(Log _log, String method) { - Long startTime = perfKeyMaps.get().get(method); - long endTime = System.currentTimeMillis(); - StringBuilder sb = new StringBuilder(""); - _log.info(sb); - } - - /** * The check here is kind of not clean. It first use a for loop to go through * all input formats, and choose the ones that extend ReworkMapredInputFormat * to a set. And finally go through the ReworkMapredInputFormat set, and call Index: ql/src/java/org/apache/hadoop/hive/ql/Driver.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/Driver.java (revision 1155439) +++ ql/src/java/org/apache/hadoop/hive/ql/Driver.java (working copy) @@ -71,6 +71,7 @@ import org.apache.hadoop.hive.ql.lockmgr.HiveLockObject; import org.apache.hadoop.hive.ql.lockmgr.LockException; import org.apache.hadoop.hive.ql.lockmgr.HiveLockObject.HiveLockObjectData; +import org.apache.hadoop.hive.ql.log.PerfLogger; import org.apache.hadoop.hive.ql.metadata.AuthorizationException; import org.apache.hadoop.hive.ql.metadata.DummyPartition; import org.apache.hadoop.hive.ql.metadata.Hive; @@ -390,8 +391,9 @@ * @return */ public int compile(String command, boolean resetTaskIds) { + PerfLogger perfLogger = PerfLogger.getPerfLogger(); + perfLogger.PerfLogBegin(LOG, PerfLogger.COMPILE); - Utilities.PerfLogBegin(LOG, "compile"); //holder for parent command type/string when executing reentrant queries QueryState queryState = new QueryState(); @@ -477,14 +479,14 @@ if (HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_AUTHORIZATION_ENABLED)) { try { - Utilities.PerfLogBegin(LOG, "doAuthorization"); + perfLogger.PerfLogBegin(LOG, PerfLogger.DO_AUTHORIZATION); doAuthorization(sem); } catch (AuthorizationException authExp) { console.printError("Authorization failed:" + authExp.getMessage() + ". Use show grant to get more details."); return 403; } finally { - Utilities.PerfLogEnd(LOG, "doAuthorization"); + perfLogger.PerfLogEnd(LOG, PerfLogger.DO_AUTHORIZATION); } } @@ -510,8 +512,8 @@ + org.apache.hadoop.util.StringUtils.stringifyException(e)); return (12); } finally { + perfLogger.PerfLogEnd(LOG, PerfLogger.COMPILE); restoreSession(queryState); - Utilities.PerfLogEnd(LOG, "compile"); } } @@ -760,9 +762,9 @@ * sure that the locks are lexicographically sorted. **/ public int acquireReadWriteLocks() { + PerfLogger perfLogger = PerfLogger.getPerfLogger(); + perfLogger.PerfLogBegin(LOG, PerfLogger.ACQUIRE_READ_WRITE_LOCKS); - Utilities.PerfLogBegin(LOG, "acquireReadWriteLocks"); - try { int sleepTime = conf.getIntVar(HiveConf.ConfVars.HIVE_LOCK_SLEEP_BETWEEN_RETRIES) * 1000; int numRetries = conf.getIntVar(HiveConf.ConfVars.HIVE_LOCK_NUMRETRIES); @@ -856,7 +858,7 @@ + org.apache.hadoop.util.StringUtils.stringifyException(e)); return (10); } finally { - Utilities.PerfLogEnd(LOG, "acquireReadWriteLocks"); + perfLogger.PerfLogEnd(LOG, PerfLogger.ACQUIRE_READ_WRITE_LOCKS); } } @@ -880,19 +882,22 @@ * locks have already been released, ignore them **/ private void releaseLocks(List hiveLocks) { - Utilities.PerfLogBegin(LOG, "releaseLocks"); + PerfLogger perfLogger = PerfLogger.getPerfLogger(); + perfLogger.PerfLogBegin(LOG, PerfLogger.RELEASE_LOCKS); if (hiveLocks != null) { ctx.getHiveLockMgr().releaseLocks(hiveLocks); } ctx.setHiveLocks(null); - Utilities.PerfLogEnd(LOG, "releaseLocks"); + perfLogger.PerfLogEnd(LOG, PerfLogger.RELEASE_LOCKS); } public CommandProcessorResponse run(String command) throws CommandNeedRetryException { errorMessage = null; SQLState = null; + // Reset the perf logger + PerfLogger.getPerfLogger(true); int ret = compile(command); if (ret != 0) { @@ -1022,7 +1027,8 @@ } public int execute() throws CommandNeedRetryException { - Utilities.PerfLogBegin(LOG, "Driver.execute"); + PerfLogger perfLogger = PerfLogger.getPerfLogger(); + perfLogger.PerfLogBegin(LOG, PerfLogger.DRIVER_EXECUTE); boolean noName = StringUtils.isEmpty(conf.getVar(HiveConf.ConfVars.HADOOPJOBNAME)); int maxlen = conf.getIntVar(HiveConf.ConfVars.HIVEJOBNAMELENGTH); @@ -1051,18 +1057,18 @@ for (Hook peh : getPreExecHooks()) { if (peh instanceof ExecuteWithHookContext) { - Utilities.PerfLogBegin(LOG, "PreHook." + peh.getClass().getSimpleName()); + perfLogger.PerfLogBegin(LOG, PerfLogger.PRE_HOOK + peh.getClass().getSimpleName()); ((ExecuteWithHookContext) peh).run(hookContext); - Utilities.PerfLogEnd(LOG, "PreHook." + peh.getClass().getSimpleName()); + perfLogger.PerfLogEnd(LOG, PerfLogger.PRE_HOOK + peh.getClass().getSimpleName()); } else if (peh instanceof PreExecute) { - Utilities.PerfLogBegin(LOG, "PreHook." + peh.getClass().getSimpleName()); + perfLogger.PerfLogBegin(LOG, PerfLogger.PRE_HOOK + peh.getClass().getSimpleName()); ((PreExecute) peh).run(SessionState.get(), plan.getInputs(), plan.getOutputs(), ShimLoader.getHadoopShims().getUGIForConf(conf)); - Utilities.PerfLogEnd(LOG, "PreHook." + peh.getClass().getSimpleName()); + perfLogger.PerfLogEnd(LOG, PerfLogger.PRE_HOOK + peh.getClass().getSimpleName()); } } @@ -1191,19 +1197,19 @@ // Get all the post execution hooks and execute them. for (Hook peh : getPostExecHooks()) { if (peh instanceof ExecuteWithHookContext) { - Utilities.PerfLogBegin(LOG, "PostHook." + peh.getClass().getSimpleName()); + perfLogger.PerfLogBegin(LOG, PerfLogger.POST_HOOK + peh.getClass().getSimpleName()); ((ExecuteWithHookContext) peh).run(hookContext); - Utilities.PerfLogEnd(LOG, "PostHook." + peh.getClass().getSimpleName()); + perfLogger.PerfLogEnd(LOG, PerfLogger.POST_HOOK + peh.getClass().getSimpleName()); } else if (peh instanceof PostExecute) { - Utilities.PerfLogBegin(LOG, "PostHook." + peh.getClass().getSimpleName()); + perfLogger.PerfLogBegin(LOG, PerfLogger.POST_HOOK + peh.getClass().getSimpleName()); ((PostExecute) peh).run(SessionState.get(), plan.getInputs(), plan.getOutputs(), (SessionState.get() != null ? SessionState.get().getLineageState().getLineageInfo() : null), ShimLoader.getHadoopShims().getUGIForConf(conf)); - Utilities.PerfLogEnd(LOG, "PostHook." + peh.getClass().getSimpleName()); + perfLogger.PerfLogEnd(LOG, PerfLogger.POST_HOOK + peh.getClass().getSimpleName()); } } @@ -1234,7 +1240,7 @@ if (noName) { conf.setVar(HiveConf.ConfVars.HADOOPJOBNAME, ""); } - Utilities.PerfLogEnd(LOG, "Driver.execute"); + perfLogger.PerfLogEnd(LOG, PerfLogger.DRIVER_EXECUTE); if (SessionState.get().getLastMapRedStatsList() != null && SessionState.get().getLastMapRedStatsList().size() > 0) {