commit aa4dbb4ab6b78f76ffc43b82a987ffc97f64d225 Author: Bharath Krishna Date: Tue Apr 10 22:03:40 2018 -0700 HIVE-19133 : HS2 WebUI phase-wise performance metrics not showing correctly. - Removing the passing of PerfLogger between handler and background threads. - Resetting PerfLogger in compile and execution phase. diff --git common/src/java/org/apache/hadoop/hive/ql/log/PerfLogger.java common/src/java/org/apache/hadoop/hive/ql/log/PerfLogger.java index 764a832e28114c977fb5cd2c0191e944500d7394..65745f211d03e0df32c02a9e0029608ba8f0515b 100644 --- common/src/java/org/apache/hadoop/hive/ql/log/PerfLogger.java +++ common/src/java/org/apache/hadoop/hive/ql/log/PerfLogger.java @@ -51,14 +51,12 @@ public static final String SERIALIZE_PLAN = "serializePlan"; public static final String DESERIALIZE_PLAN = "deserializePlan"; public static final String CLONE_PLAN = "clonePlan"; - public static final String TASK = "task."; public static final String RELEASE_LOCKS = "releaseLocks"; public static final String PRUNE_LISTING = "prune-listing"; public static final String PARTITION_RETRIEVING = "partition-retrieving"; public static final String PRE_HOOK = "PreHook."; public static final String POST_HOOK = "PostHook."; public static final String FAILURE_HOOK = "FailureHook."; - public static final String DRIVER_RUN = "Driver.run"; public static final String TEZ_COMPILER = "TezCompiler"; public static final String TEZ_SUBMIT_TO_RUNNING = "TezSubmitToRunningDag"; public static final String TEZ_BUILD_DAG = "TezBuildDag"; diff --git itests/hive-unit/src/test/java/org/apache/hive/service/cli/session/TestQueryDisplay.java itests/hive-unit/src/test/java/org/apache/hive/service/cli/session/TestQueryDisplay.java index b51523fd2574c5b4b8c1dfe114b4e23bd93f2f1a..6e8c5cff5bb97a23c9de63137357acb96d0d44ba 100644 --- itests/hive-unit/src/test/java/org/apache/hive/service/cli/session/TestQueryDisplay.java +++ itests/hive-unit/src/test/java/org/apache/hive/service/cli/session/TestQueryDisplay.java @@ -20,6 +20,7 @@ import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.QueryDisplay; import org.apache.hadoop.hive.ql.QueryInfo; +import org.apache.hadoop.hive.ql.log.PerfLogger; import org.apache.hadoop.hive.ql.plan.api.StageType; import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hive.service.cli.OperationHandle; @@ -153,6 +154,12 @@ private void verifyDDL(QueryInfo queryInfo, String stmt, String handle, boolean Assert.assertTrue(qDisplay1.getPerfLogStarts(QueryDisplay.Phase.COMPILATION).size() > 0); Assert.assertTrue(qDisplay1.getPerfLogEnds(QueryDisplay.Phase.COMPILATION).size() > 0); + Assert.assertTrue(qDisplay1.getPerfLogStarts(QueryDisplay.Phase.COMPILATION).containsKey(PerfLogger.COMPILE)); + Assert.assertFalse(qDisplay1.getPerfLogStarts(QueryDisplay.Phase.EXECUTION).containsKey(PerfLogger.COMPILE)); + Assert.assertTrue(qDisplay1.getPerfLogStarts(QueryDisplay.Phase.EXECUTION).containsKey(PerfLogger.DRIVER_EXECUTE)); + Assert.assertFalse(qDisplay1.getPerfLogStarts(QueryDisplay.Phase.COMPILATION) + .containsKey(PerfLogger.DRIVER_EXECUTE)); + Assert.assertEquals(qDisplay1.getTaskDisplays().size(), 1); QueryDisplay.TaskDisplay tInfo1 = qDisplay1.getTaskDisplays().get(0); Assert.assertEquals(tInfo1.getTaskId(), "Stage-0"); diff --git ql/src/java/org/apache/hadoop/hive/ql/Driver.java ql/src/java/org/apache/hadoop/hive/ql/Driver.java index a88453c97835db847d74b4b4c3ef318d4d6c0ce5..210e96d4bef407327d97cd1b03992bf75344d12e 100644 --- ql/src/java/org/apache/hadoop/hive/ql/Driver.java +++ ql/src/java/org/apache/hadoop/hive/ql/Driver.java @@ -488,8 +488,7 @@ public int compile(String command, boolean resetTaskIds) { // interrupted, it should be set to true if the compile is called within another method like // runInternal, which defers the close to the called in that method. private void compile(String command, boolean resetTaskIds, boolean deferClose) throws CommandProcessorResponse { - PerfLogger perfLogger = SessionState.getPerfLogger(true); - perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.DRIVER_RUN); + PerfLogger perfLogger = SessionState.getPerfLogger(); perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.COMPILE); lDrvState.stateLock.lock(); try { @@ -659,7 +658,7 @@ public void run() { // get the output schema schema = getSchema(sem, conf); - plan = new QueryPlan(queryStr, sem, perfLogger.getStartTime(PerfLogger.DRIVER_RUN), queryId, + plan = new QueryPlan(queryStr, sem, queryDisplay.getQueryStartTime(), queryId, queryState.getHiveOperation(), schema); @@ -1637,7 +1636,7 @@ private void compileInternal(String command, boolean deferClose) throws CommandP metrics.incrementCounter(MetricsConstant.WAITING_COMPILE_OPS, 1); } - PerfLogger perfLogger = SessionState.getPerfLogger(); + PerfLogger perfLogger = SessionState.getPerfLogger(true); perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.WAIT_COMPILE); final ReentrantLock compileLock = tryAcquireCompileLock(isParallelEnabled, command); @@ -1663,7 +1662,6 @@ private void compileInternal(String command, boolean deferClose) throws CommandP } finally { compileLock.unlock(); } - //Save compile-time PerfLogging for WebUI. //Execution-time Perf logs are done by either another thread's PerfLogger //or a reset PerfLogger. @@ -1771,19 +1769,18 @@ private void runInternal(String command, boolean alreadyCompiled) throws Command throw createProcessorResponse(12); } - PerfLogger perfLogger = null; - if (!alreadyCompiled) { // compile internal will automatically reset the perf logger compileInternal(command, true); - // then we continue to use this perf logger - perfLogger = SessionState.getPerfLogger(); } else { - // reuse existing perf logger. - perfLogger = SessionState.getPerfLogger(); // Since we're reusing the compiled plan, we need to update its start time for current run - plan.setQueryStartTime(perfLogger.getStartTime(PerfLogger.DRIVER_RUN)); + plan.setQueryStartTime(queryDisplay.getQueryStartTime()); } + + //Reset the PerfLogger so that it doesn't retain any previous values. + // Any value from compilation phase can be obtained through the map set in queryDisplay during compilation. + PerfLogger perfLogger = SessionState.getPerfLogger(true); + // the reason that we set the txn manager for the cxt here is because each // query has its own ctx object. The txn mgr is shared across the // same instance of Driver, which can run multiple queries. @@ -1816,7 +1813,6 @@ else if(plan.getOperation() == HiveOperation.ROLLBACK) { throw handleHiveException(e, 12); } - perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.DRIVER_RUN); queryDisplay.setPerfLogStarts(QueryDisplay.Phase.EXECUTION, perfLogger.getStartTimes()); queryDisplay.setPerfLogEnds(QueryDisplay.Phase.EXECUTION, perfLogger.getEndTimes()); diff --git service/src/java/org/apache/hive/service/cli/operation/SQLOperation.java service/src/java/org/apache/hive/service/cli/operation/SQLOperation.java index 5fc935be571ac974de29928d3fa85f6a93c521cb..85f92d284534b17faa35cd11e1d9961f404d2d04 100644 --- service/src/java/org/apache/hive/service/cli/operation/SQLOperation.java +++ service/src/java/org/apache/hive/service/cli/operation/SQLOperation.java @@ -263,8 +263,9 @@ public void runInternal() throws HiveSQLException { // 1) ThreadLocal Hive object needs to be set in background thread // 2) The metastore client in Hive is associated with right user. // 3) Current UGI will get used by metastore when metastore is in embedded mode - Runnable work = new BackgroundWork(getCurrentUGI(), parentSession.getSessionHive(), - SessionState.getPerfLogger(), SessionState.get(), asyncPrepare); + Runnable work = + new BackgroundWork(getCurrentUGI(), parentSession.getSessionHive(), SessionState.get(), + asyncPrepare); try { // This submit blocks if no background threads are available to run this operation @@ -282,16 +283,14 @@ public void runInternal() throws HiveSQLException { private final class BackgroundWork implements Runnable { private final UserGroupInformation currentUGI; private final Hive parentHive; - private final PerfLogger parentPerfLogger; private final SessionState parentSessionState; private final boolean asyncPrepare; private BackgroundWork(UserGroupInformation currentUGI, - Hive parentHive, PerfLogger parentPerfLogger, + Hive parentHive, SessionState parentSessionState, boolean asyncPrepare) { this.currentUGI = currentUGI; this.parentHive = parentHive; - this.parentPerfLogger = parentPerfLogger; this.parentSessionState = parentSessionState; this.asyncPrepare = asyncPrepare; } @@ -304,7 +303,7 @@ public Object run() throws HiveSQLException { Hive.set(parentHive); // TODO: can this result in cross-thread reuse of session state? SessionState.setCurrentSessionState(parentSessionState); - PerfLogger.setPerfLogger(parentPerfLogger); + PerfLogger.setPerfLogger(SessionState.getPerfLogger()); LogUtils.registerLoggingContext(queryState.getConf()); try { if (asyncPrepare) {