commit 95d2d9c5b4b4076c8bbfa0c352ae71b56ef037e2 Author: Sahil Takiar Date: Wed Jan 10 21:47:58 2018 -0800 HIVE-18436: Upgrade to Spark 2.3.0 diff --git a/itests/pom.xml b/itests/pom.xml index a782cd27d0..40bb0548c7 100644 --- a/itests/pom.xml +++ b/itests/pom.xml @@ -99,10 +99,10 @@ cd - fi tar -zxf $DOWNLOAD_DIR/$tarName -C $BASE_DIR - mv $BASE_DIR/spark-${spark.version}-bin-hadoop2-without-hive $BASE_DIR/$finalName + mv $BASE_DIR/spark-${spark.version}-bin-hadoop3-beta1-without-hive $BASE_DIR/$finalName } mkdir -p $DOWNLOAD_DIR - download "http://d3jw87u4immizc.cloudfront.net/spark-tarball/spark-${spark.version}-bin-hadoop2-without-hive.tgz" "spark" + download "http://d3jw87u4immizc.cloudfront.net/spark-tarball/spark-${spark.version}-bin-hadoop3-beta1-without-hive.tgz" "spark" cp -f $HIVE_ROOT/data/conf/spark/log4j2.properties $BASE_DIR/spark/conf/ diff --git a/pom.xml b/pom.xml index a242fbf203..5be30f6696 100644 --- a/pom.xml +++ b/pom.xml @@ -185,7 +185,7 @@ 1.4.3 1.10.19 2.0.0-M5 - 4.0.52.Final + 4.1.17.Final 3.10.5.Final 1.9.0 0.16.0 @@ -198,7 +198,7 @@ 0.9.1 0.92.0-incubating 2.2.0 - 2.2.0 + 2.3.0 2.11 2.11.8 1.1 diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/LocalHiveSparkClient.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/LocalHiveSparkClient.java index f43b44968b..dff3b0f569 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/LocalHiveSparkClient.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/LocalHiveSparkClient.java @@ -104,7 +104,7 @@ private LocalHiveSparkClient(SparkConf sparkConf, HiveConf hiveConf) sc.addJar(regJar); } jobMetricsListener = new JobMetricsListener(); - sc.sc().listenerBus().addListener(jobMetricsListener); + sc.sc().addSparkListener(jobMetricsListener); } @Override diff --git a/spark-client/src/test/java/org/apache/hive/spark/client/rpc/TestKryoMessageCodec.java b/spark-client/src/test/java/org/apache/hive/spark/client/rpc/TestKryoMessageCodec.java index 24858d7cef..fb736471b2 100644 --- a/spark-client/src/test/java/org/apache/hive/spark/client/rpc/TestKryoMessageCodec.java +++ b/spark-client/src/test/java/org/apache/hive/spark/client/rpc/TestKryoMessageCodec.java @@ -72,7 +72,8 @@ public void testEmbeddedChannel() throws Exception { c.writeAndFlush(MESSAGE); assertEquals(1, c.outboundMessages().size()); assertFalse(MESSAGE.getClass().equals(c.outboundMessages().peek().getClass())); - c.writeInbound(c.readOutbound()); + Object readOutboundResult = c.readOutbound(); + c.writeInbound(readOutboundResult); assertEquals(1, c.inboundMessages().size()); assertEquals(MESSAGE, c.readInbound()); c.close(); diff --git a/spark-client/src/test/java/org/apache/hive/spark/client/rpc/TestRpc.java b/spark-client/src/test/java/org/apache/hive/spark/client/rpc/TestRpc.java index 21e4595806..fafdff77ea 100644 --- a/spark-client/src/test/java/org/apache/hive/spark/client/rpc/TestRpc.java +++ b/spark-client/src/test/java/org/apache/hive/spark/client/rpc/TestRpc.java @@ -323,7 +323,8 @@ private void transfer(Rpc serverRpc, Rpc clientRpc) { int count = 0; while (!client.outboundMessages().isEmpty()) { - server.writeInbound(client.readOutbound()); + Object readOutboundResult = client.readOutbound(); + server.writeInbound(readOutboundResult); count++; } server.flush(); @@ -331,7 +332,8 @@ private void transfer(Rpc serverRpc, Rpc clientRpc) { count = 0; while (!server.outboundMessages().isEmpty()) { - client.writeInbound(server.readOutbound()); + Object readOutboundResult = server.readOutbound(); + client.writeInbound(readOutboundResult); count++; } client.flush();