diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/RemoteSparkJobMonitor.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/RemoteSparkJobMonitor.java index bdb1527..77038fc 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/RemoteSparkJobMonitor.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/RemoteSparkJobMonitor.java @@ -116,7 +116,29 @@ public int startMonitor() { done = true; break; case FAILED: - console.printError("Status: Failed"); + String detail = sparkJobStatus.getError().getMessage(); + StringBuilder errBuilder = new StringBuilder(); + errBuilder.append("Job failed with "); + if (detail == null) { + errBuilder.append("UNKNOWN reason"); + } else { + // We SerDe the Throwable as String, parse it for the root cause + final String CAUSE_CAPTION = "Caused by: "; + int index = detail.lastIndexOf(CAUSE_CAPTION); + if (index != -1) { + String rootCause = detail.substring(index + CAUSE_CAPTION.length()); + index = rootCause.indexOf(System.getProperty("line.separator")); + if (index != -1) { + errBuilder.append(rootCause.substring(0, index)); + } else { + errBuilder.append(rootCause); + } + } else { + errBuilder.append(detail); + } + detail = System.getProperty("line.separator") + detail; + } + console.printError(errBuilder.toString(), detail); running = false; done = true; rc = 3; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/SparkJobStatus.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/SparkJobStatus.java index 7959089..72ce439 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/SparkJobStatus.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/SparkJobStatus.java @@ -44,4 +44,6 @@ SparkStatistics getSparkStatistics(); void cleanup(); + + Throwable getError(); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/impl/LocalSparkJobStatus.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/impl/LocalSparkJobStatus.java index 4e93979..a94d4ed 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/impl/LocalSparkJobStatus.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/impl/LocalSparkJobStatus.java @@ -159,6 +159,18 @@ public void cleanup() { } } + @Override + public Throwable getError() { + if (future.isDone()) { + try { + future.get(); + } catch (Throwable e) { + return e; + } + } + return null; + } + private SparkJobInfo getJobInfo() { return sparkContext.statusTracker().getJobInfo(jobId); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/impl/RemoteSparkJobStatus.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/impl/RemoteSparkJobStatus.java index 9fc717f..e87a21a 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/impl/RemoteSparkJobStatus.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/impl/RemoteSparkJobStatus.java @@ -136,6 +136,11 @@ public void cleanup() { } + @Override + public Throwable getError() { + return jobHandle.getError(); + } + private SparkJobInfo getSparkJobInfo() throws HiveException { Integer sparkJobId = jobHandle.getSparkJobIds().size() == 1 ? jobHandle.getSparkJobIds().get(0) : null; diff --git a/spark-client/src/main/java/org/apache/hive/spark/client/JobHandle.java b/spark-client/src/main/java/org/apache/hive/spark/client/JobHandle.java index c02c403..dffd60c 100644 --- a/spark-client/src/main/java/org/apache/hive/spark/client/JobHandle.java +++ b/spark-client/src/main/java/org/apache/hive/spark/client/JobHandle.java @@ -61,6 +61,11 @@ State getState(); /** + * Return the error if the job has failed. + */ + Throwable getError(); + + /** * The current state of the submitted job. */ static enum State { diff --git a/spark-client/src/main/java/org/apache/hive/spark/client/JobHandleImpl.java b/spark-client/src/main/java/org/apache/hive/spark/client/JobHandleImpl.java index 7645702..2881252 100644 --- a/spark-client/src/main/java/org/apache/hive/spark/client/JobHandleImpl.java +++ b/spark-client/src/main/java/org/apache/hive/spark/client/JobHandleImpl.java @@ -128,6 +128,11 @@ public State getState() { return state; } + @Override + public Throwable getError() { + return promise.cause(); + } + public void setSparkCounters(SparkCounters sparkCounters) { this.sparkCounters = sparkCounters; }