diff --git a/pom.xml b/pom.xml index 75a41a4..fb2ab9f 100644 --- a/pom.xml +++ b/pom.xml @@ -155,7 +155,7 @@ 4.0.4 0.5.2 2.2.0 - 1.2.0 + 1.3.0 2.10 2.10.4 1.1 @@ -215,6 +215,16 @@ false + + spark-1.3 + http://ec2-50-18-79-139.us-west-1.compute.amazonaws.com/data/spark_2.10-1.3-rc1/ + + true + + + false + + diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/impl/JobMetricsListener.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/impl/JobMetricsListener.java index 8243d4d..51772cd 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/impl/JobMetricsListener.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/impl/JobMetricsListener.java @@ -39,6 +39,8 @@ import org.apache.spark.scheduler.SparkListenerTaskGettingResult; import org.apache.spark.scheduler.SparkListenerTaskStart; import org.apache.spark.scheduler.SparkListenerUnpersistRDD; +import org.apache.spark.scheduler.SparkListenerExecutorRemoved; +import org.apache.spark.scheduler.SparkListenerExecutorAdded; import com.google.common.collect.Lists; import com.google.common.collect.Maps; @@ -52,6 +54,16 @@ private final Map>> allJobMetrics = Maps.newHashMap(); @Override + public void onExecutorRemoved(SparkListenerExecutorRemoved removed) { + + } + + @Override + public void onExecutorAdded(SparkListenerExecutorAdded added) { + + } + + @Override public void onStageCompleted(SparkListenerStageCompleted stageCompleted) { } diff --git a/spark-client/src/main/java/org/apache/hive/spark/client/RemoteDriver.java b/spark-client/src/main/java/org/apache/hive/spark/client/RemoteDriver.java index 8bda02b..c2ac0c2 100644 --- a/spark-client/src/main/java/org/apache/hive/spark/client/RemoteDriver.java +++ b/spark-client/src/main/java/org/apache/hive/spark/client/RemoteDriver.java @@ -57,6 +57,8 @@ import org.apache.spark.scheduler.SparkListenerTaskGettingResult; import org.apache.spark.scheduler.SparkListenerTaskStart; import org.apache.spark.scheduler.SparkListenerUnpersistRDD; +import org.apache.spark.scheduler.SparkListenerExecutorRemoved; +import org.apache.spark.scheduler.SparkListenerExecutorAdded; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -425,6 +427,16 @@ private void monitorJob(JavaFutureAction job, private final Map stageToJobId = Maps.newHashMap(); @Override + public void onExecutorRemoved(SparkListenerExecutorRemoved removed) { + + } + + @Override + public void onExecutorAdded(SparkListenerExecutorAdded added) { + + } + + @Override public void onJobStart(SparkListenerJobStart jobStart) { synchronized (stageToJobId) { for (int i = 0; i < jobStart.stageIds().length(); i++) {