commit b24b5cff443714d0e42169c09488364b07e4531c Author: Bharath Krishna Date: Tue Apr 10 22:03:40 2018 -0700 HIVE-19133 : HS2 WebUI phase-wise performance metrics not showing correctly Removing the passing of PerfLogger between handler and background threads. Resetting PerfLogger in compile and execution phase. diff --git ql/src/java/org/apache/hadoop/hive/ql/Driver.java ql/src/java/org/apache/hadoop/hive/ql/Driver.java index a88453c97835db847d74b4b4c3ef318d4d6c0ce5..41c16bdfc96571b74ce53f369ce6781e0a9de7ca 100644 --- ql/src/java/org/apache/hadoop/hive/ql/Driver.java +++ ql/src/java/org/apache/hadoop/hive/ql/Driver.java @@ -488,8 +488,7 @@ public int compile(String command, boolean resetTaskIds) { // interrupted, it should be set to true if the compile is called within another method like // runInternal, which defers the close to the called in that method. private void compile(String command, boolean resetTaskIds, boolean deferClose) throws CommandProcessorResponse { - PerfLogger perfLogger = SessionState.getPerfLogger(true); - perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.DRIVER_RUN); + PerfLogger perfLogger = SessionState.getPerfLogger(); perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.COMPILE); lDrvState.stateLock.lock(); try { @@ -659,7 +658,7 @@ public void run() { // get the output schema schema = getSchema(sem, conf); - plan = new QueryPlan(queryStr, sem, perfLogger.getStartTime(PerfLogger.DRIVER_RUN), queryId, + plan = new QueryPlan(queryStr, sem, queryDisplay.getQueryStartTime(), queryId, queryState.getHiveOperation(), schema); @@ -1637,7 +1636,8 @@ private void compileInternal(String command, boolean deferClose) throws CommandP metrics.incrementCounter(MetricsConstant.WAITING_COMPILE_OPS, 1); } - PerfLogger perfLogger = SessionState.getPerfLogger(); + PerfLogger perfLogger = SessionState.getPerfLogger(true); + perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.DRIVER_RUN); perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.WAIT_COMPILE); final ReentrantLock compileLock = tryAcquireCompileLock(isParallelEnabled, command); @@ -1663,7 +1663,7 @@ private void compileInternal(String command, boolean deferClose) throws CommandP } finally { compileLock.unlock(); } - + perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.DRIVER_RUN); //Save compile-time PerfLogging for WebUI. //Execution-time Perf logs are done by either another thread's PerfLogger //or a reset PerfLogger. @@ -1771,19 +1771,20 @@ private void runInternal(String command, boolean alreadyCompiled) throws Command throw createProcessorResponse(12); } - PerfLogger perfLogger = null; - if (!alreadyCompiled) { // compile internal will automatically reset the perf logger compileInternal(command, true); - // then we continue to use this perf logger - perfLogger = SessionState.getPerfLogger(); } else { - // reuse existing perf logger. - perfLogger = SessionState.getPerfLogger(); // Since we're reusing the compiled plan, we need to update its start time for current run - plan.setQueryStartTime(perfLogger.getStartTime(PerfLogger.DRIVER_RUN)); + plan.setQueryStartTime(queryDisplay.getQueryStartTime()); } + + //Reset the PerfLogger so that it doesn't retain any previous values. + // Any value from compilation phase can be obtained through the map set in queryDisplay during compilation. + PerfLogger perfLogger = SessionState.getPerfLogger(true); + + perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.DRIVER_RUN); + // the reason that we set the txn manager for the cxt here is because each // query has its own ctx object. The txn mgr is shared across the // same instance of Driver, which can run multiple queries. diff --git service/src/java/org/apache/hive/service/cli/operation/SQLOperation.java service/src/java/org/apache/hive/service/cli/operation/SQLOperation.java index 5fc935be571ac974de29928d3fa85f6a93c521cb..85f92d284534b17faa35cd11e1d9961f404d2d04 100644 --- service/src/java/org/apache/hive/service/cli/operation/SQLOperation.java +++ service/src/java/org/apache/hive/service/cli/operation/SQLOperation.java @@ -263,8 +263,9 @@ public void runInternal() throws HiveSQLException { // 1) ThreadLocal Hive object needs to be set in background thread // 2) The metastore client in Hive is associated with right user. // 3) Current UGI will get used by metastore when metastore is in embedded mode - Runnable work = new BackgroundWork(getCurrentUGI(), parentSession.getSessionHive(), - SessionState.getPerfLogger(), SessionState.get(), asyncPrepare); + Runnable work = + new BackgroundWork(getCurrentUGI(), parentSession.getSessionHive(), SessionState.get(), + asyncPrepare); try { // This submit blocks if no background threads are available to run this operation @@ -282,16 +283,14 @@ public void runInternal() throws HiveSQLException { private final class BackgroundWork implements Runnable { private final UserGroupInformation currentUGI; private final Hive parentHive; - private final PerfLogger parentPerfLogger; private final SessionState parentSessionState; private final boolean asyncPrepare; private BackgroundWork(UserGroupInformation currentUGI, - Hive parentHive, PerfLogger parentPerfLogger, + Hive parentHive, SessionState parentSessionState, boolean asyncPrepare) { this.currentUGI = currentUGI; this.parentHive = parentHive; - this.parentPerfLogger = parentPerfLogger; this.parentSessionState = parentSessionState; this.asyncPrepare = asyncPrepare; } @@ -304,7 +303,7 @@ public Object run() throws HiveSQLException { Hive.set(parentHive); // TODO: can this result in cross-thread reuse of session state? SessionState.setCurrentSessionState(parentSessionState); - PerfLogger.setPerfLogger(parentPerfLogger); + PerfLogger.setPerfLogger(SessionState.getPerfLogger()); LogUtils.registerLoggingContext(queryState.getConf()); try { if (asyncPrepare) {