commit 4ec14de3fbe1244ef251230f7466dc2c7ab1ba76 Author: Sahil Takiar Date: Fri Feb 9 16:08:25 2018 -0800 HIVE-18672: Printed state in RemoteSparkJobMonitor is ambiguous diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/RemoteSparkJobMonitor.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/RemoteSparkJobMonitor.java index fc4e4de2f9..f630f7fba7 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/RemoteSparkJobMonitor.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/RemoteSparkJobMonitor.java @@ -62,10 +62,11 @@ public int startMonitor() { perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.SPARK_SUBMIT_TO_RUNNING); startTime = System.currentTimeMillis(); + JobHandle.State state = null; while (true) { try { - JobHandle.State state = sparkJobStatus.getRemoteJobState(); + state = sparkJobStatus.getRemoteJobState(); Preconditions.checkState(sparkJobStatus.isRemoteActive(), "Connection to remote Spark driver was lost"); switch (state) { @@ -76,14 +77,13 @@ public int startMonitor() { HiveException he = new HiveException(ErrorMsg.SPARK_JOB_MONITOR_TIMEOUT, Long.toString(timeCount)); console.printError(he.getMessage()); - console.printError("Status: " + state); sparkJobStatus.setError(he); running = false; done = true; rc = 2; } if (LOG.isDebugEnabled()) { - console.printInfo("state = " + state); + console.printInfo("Spark job state = " + state ); } break; case STARTED: @@ -97,8 +97,7 @@ public int startMonitor() { console.printInfo("\nQuery Hive on Spark job[" + sparkJobStatus.getJobId() + "] stages: " + Arrays.toString(sparkJobStatus.getStageIds())); - console.printInfo("\nStatus: Running (Hive on Spark job[" - + sparkJobStatus.getJobId() + "])"); + console.printInfo("Spark job[" + sparkJobStatus.getJobId() + "] status = RUNNING"); running = true; String format = "Job Progress Format\nCurrentTime StageId_StageAttemptId: " @@ -141,8 +140,8 @@ public int startMonitor() { printStatus(progressMap, lastProgressMap); lastProgressMap = progressMap; double duration = (System.currentTimeMillis() - startTime) / 1000.0; - console.printInfo("Status: Finished successfully in " - + String.format("%.2f seconds", duration)); + console.printInfo("Spark job[" + sparkJobStatus.getJobId() + "] finished successfully in " + + String.format("%.2f second(s)", duration)); running = false; done = true; break; @@ -175,7 +174,7 @@ public int startMonitor() { rc = 3; break; case CANCELLED: - console.printInfo("Status: Cancelled"); + console.printInfo("Spark job[" + sparkJobStatus.getJobId() + " was cancelled"); running = false; done = true; rc = 3; @@ -192,7 +191,8 @@ public int startMonitor() { finalException = new HiveException(e, ErrorMsg.SPARK_JOB_INTERRUPTED); LOG.warn("Interrupted while monitoring the Hive on Spark application, exiting"); } else { - String msg = " with exception '" + Utilities.getNameMessage(e) + "'"; + String msg = " with exception '" + Utilities.getNameMessage(e) + "' Last known state = " + + (state != null ? state.name() : "UNKNOWN"); msg = "Failed to monitor Job[" + sparkJobStatus.getJobId() + "]" + msg; // Has to use full name to make sure it does not conflict with diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/SparkJobMonitor.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/SparkJobMonitor.java index b3e49fe0d6..7afd886407 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/SparkJobMonitor.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/SparkJobMonitor.java @@ -82,7 +82,7 @@ protected SparkJobMonitor(HiveConf hiveConf) { monitorTimeoutInterval = hiveConf.getTimeVar( HiveConf.ConfVars.SPARK_JOB_MONITOR_TIMEOUT, TimeUnit.SECONDS); inPlaceUpdate = InPlaceUpdate.canRenderInPlace(hiveConf) && !SessionState.getConsole().getIsSilent(); - console = SessionState.getConsole(); + console = new SessionState.LogHelper(LOG); out = SessionState.LogHelper.getInfoStream(); }