diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkTask.java ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkTask.java index 1342afe..5e5ac53 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkTask.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkTask.java @@ -52,6 +52,7 @@ import org.apache.hadoop.hive.ql.exec.spark.status.LocalSparkJobMonitor; import org.apache.hadoop.hive.ql.exec.spark.status.SparkJobRef; import org.apache.hadoop.hive.ql.exec.spark.status.SparkJobStatus; +import org.apache.hadoop.hive.ql.history.HiveHistory.Keys; import org.apache.hadoop.hive.ql.log.PerfLogger; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.metadata.Partition; @@ -66,6 +67,7 @@ import org.apache.hadoop.hive.ql.plan.SparkWork; import org.apache.hadoop.hive.ql.plan.StatsWork; import org.apache.hadoop.hive.ql.plan.api.StageType; +import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hadoop.hive.ql.stats.StatsFactory; import org.apache.hadoop.util.StringUtils; import org.apache.hive.spark.counter.SparkCounters; @@ -101,6 +103,7 @@ public int execute(DriverContext driverContext) { SparkJobRef jobRef = sparkSession.submit(driverContext, sparkWork); perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.SPARK_SUBMIT_JOB); + jobInfo(jobRef); rc = jobRef.monitorJob(); SparkJobStatus sparkJobStatus = jobRef.getSparkJobStatus(); if (rc == 0) { @@ -137,6 +140,14 @@ public int execute(DriverContext driverContext) { return rc; } + private void jobInfo(SparkJobRef jobRef) { + console.printInfo("Starting Spark Job = " + jobRef.getJobId()); + if (SessionState.get() != null) { + SessionState.get().getHiveHistory() + .setQueryProperty(SessionState.get().getQueryId(), Keys.SPARK_JOB_ID, jobRef.getJobId()); + } + } + private void logSparkStatistic(SparkStatistics sparkStatistic) { Iterator groupIterator = sparkStatistic.getStatisticGroups(); while (groupIterator.hasNext()) { diff --git ql/src/java/org/apache/hadoop/hive/ql/history/HiveHistory.java ql/src/java/org/apache/hadoop/hive/ql/history/HiveHistory.java index 7b0d978..45cd533 100644 --- ql/src/java/org/apache/hadoop/hive/ql/history/HiveHistory.java +++ ql/src/java/org/apache/hadoop/hive/ql/history/HiveHistory.java @@ -68,7 +68,8 @@ TASK_COUNTERS, TASK_NUM_MAPPERS, TASK_NUM_REDUCERS, - ROWS_INSERTED + ROWS_INSERTED, + SPARK_JOB_ID }; /**