Index: conf/hive-default.xml
===================================================================
--- conf/hive-default.xml (revision 1155439)
+++ conf/hive-default.xml (working copy)
@@ -1138,4 +1138,10 @@
by record readers
+
+ hive.exec.perf.logger
+ org.apache.hadoop.hive.ql.log.PerfLogger
+ The class responsible logging client side performance metrics. Must be a subclass of org.apache.hadoop.hive.ql.log.PerfLogger
+
+
Index: common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
===================================================================
--- common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (revision 1155439)
+++ common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (working copy)
@@ -457,6 +457,10 @@
HIVE_MAPPER_CANNOT_SPAN_MULTIPLE_PARTITIONS("hive.mapper.cannot.span.multiple.partitions", false),
HIVE_REWORK_MAPREDWORK("hive.rework.mapredwork", false),
HIVE_CONCATENATE_CHECK_INDEX ("hive.exec.concatenate.check.index", true),
+
+ // The class responsible for logging client side performance metrics
+ // Must be a subclass of org.apache.hadoop.hive.ql.log.PerfLogger
+ HIVE_PERF_LOGGER("hive.exec.perf.logger", "org.apache.hadoop.hive.ql.log.PerfLogger"),
;
public final String varname;
Index: ql/src/java/org/apache/hadoop/hive/ql/log/PerfLogger.java
===================================================================
--- ql/src/java/org/apache/hadoop/hive/ql/log/PerfLogger.java (revision 0)
+++ ql/src/java/org/apache/hadoop/hive/ql/log/PerfLogger.java (revision 0)
@@ -0,0 +1,105 @@
+package org.apache.hadoop.hive.ql.log;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
+import org.apache.hadoop.hive.ql.session.SessionState;
+import org.apache.hadoop.util.ReflectionUtils;
+
+/**
+ * PerfLogger.
+ *
+ * Can be used to measure and log the time spent by a piece of code.
+ */
+public class PerfLogger {
+ public static final String ACQUIRE_READ_WRITE_LOCKS = "acquireReadWriteLocks";
+ public static final String COMPILE = "compile";
+ public static final String DO_AUTHORIZATION = "doAuthorization";
+ public static final String DRIVER_EXECUTE = "Driver.execute";
+ public static final String RELEASE_LOCKS = "releaseLocks";
+ public static final String PRUNE_LISTING = "prune-listing";
+ public static final String PARTITION_RETRIEVING = "partition-retrieving";
+ public static final String PRE_HOOK = "PreHook.";
+ public static final String POST_HOOK = "PostHook.";
+
+ protected static final ThreadLocal perfLogger = new ThreadLocal();
+
+ protected final Map perfKeyMaps = new HashMap();
+
+ static final private Log LOG = LogFactory.getLog(PerfLogger.class.getName());
+
+ protected PerfLogger() {
+ // Use getPerfLogger to get an instance of PerfLogger
+ }
+
+ public static PerfLogger getPerfLogger() {
+ return getPerfLogger(false);
+ }
+
+ /**
+ * Call this function to get an instance of PerfLogger.
+ *
+ * Use resetPerfLogger to require a new instance. Useful at the beginning of execution.
+ *
+ * @return Tries to return an instance of the class whose name is configured in
+ * hive.exec.perf.logger, but if it can't it just returns an instance of
+ * the base PerfLogger class
+ */
+ public static PerfLogger getPerfLogger(boolean resetPerfLogger) {
+ if (perfLogger.get() == null || resetPerfLogger) {
+ if (SessionState.get() == null) {
+ perfLogger.set(new PerfLogger());
+ } else {
+ HiveConf conf = SessionState.get().getConf();
+ try {
+ perfLogger.set((PerfLogger) ReflectionUtils.newInstance(conf.getClassByName(
+ conf.getVar(ConfVars.HIVE_PERF_LOGGER)), conf));
+ } catch (ClassNotFoundException e) {
+ LOG.error("Performance Logger Class not found:" + e.getMessage());
+ perfLogger.set(new PerfLogger());
+ }
+ }
+ }
+ return perfLogger.get();
+ }
+
+ /**
+ * Call this function when you start to measure time spent by a piece of code.
+ * @param _log the logging object to be used.
+ * @param method method or ID that identifies this perf log element.
+ */
+ public void PerfLogBegin(Log _log, String method) {
+ long startTime = System.currentTimeMillis();
+ _log.info("");
+ perfKeyMaps.put(method, new Long(startTime));
+ }
+
+ /**
+ * Call this function in correspondence of PerfLogBegin to mark the end of the measurement.
+ * @param _log
+ * @param method
+ * @return long duration the difference between now and startTime, or -1 if startTime is null
+ */
+ public long PerfLogEnd(Log _log, String method) {
+ Long startTime = perfKeyMaps.get(method);
+ long endTime = System.currentTimeMillis();
+ long duration = -1;
+ StringBuilder sb = new StringBuilder("");
+ _log.info(sb);
+
+ return duration;
+ }
+}
Index: ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java
===================================================================
--- ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java (revision 1155439)
+++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java (working copy)
@@ -44,6 +44,7 @@
import org.apache.hadoop.hive.ql.lib.NodeProcessor;
import org.apache.hadoop.hive.ql.lib.Rule;
import org.apache.hadoop.hive.ql.lib.RuleRegExp;
+import org.apache.hadoop.hive.ql.log.PerfLogger;
import org.apache.hadoop.hive.ql.metadata.Hive;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.metadata.Partition;
@@ -320,8 +321,10 @@
List trueNames = null;
List unknNames = null;
- Utilities.PerfLogBegin(LOG, "prune-listing");
+ PerfLogger perfLogger = PerfLogger.getPerfLogger();
+ perfLogger.PerfLogBegin(LOG, PerfLogger.PRUNE_LISTING);
+
List partNames = Hive.get().getPartitionNames(tab.getDbName(),
tab.getTableName(), (short) -1);
@@ -366,9 +369,9 @@
LOG.debug("retained partition: " + partName);
}
}
- Utilities.PerfLogEnd(LOG, "prune-listing");
+ perfLogger.PerfLogEnd(LOG, PerfLogger.PRUNE_LISTING);
- Utilities.PerfLogBegin(LOG, "partition-retrieving");
+ perfLogger.PerfLogBegin(LOG, PerfLogger.PARTITION_RETRIEVING);
if (trueNames != null) {
List parts = Hive.get().getPartitionsByNames(tab, trueNames);
true_parts.addAll(parts);
@@ -377,7 +380,7 @@
List parts = Hive.get().getPartitionsByNames(tab, unknNames);
unkn_parts.addAll(parts);
}
- Utilities.PerfLogEnd(LOG, "partition-retrieving");
+ perfLogger.PerfLogEnd(LOG, PerfLogger.PARTITION_RETRIEVING);
}
/**
Index: ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
===================================================================
--- ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java (revision 1155439)
+++ ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java (working copy)
@@ -2004,43 +2004,7 @@
return false;
}
- private static ThreadLocal