diff --git a/itests/pom.xml b/itests/pom.xml index 40e7b9b..e1e88f6 100644 --- a/itests/pom.xml +++ b/itests/pom.xml @@ -90,7 +90,7 @@ mv $BASE_DIR/${finalName}* $BASE_DIR/$finalName } mkdir -p $DOWNLOAD_DIR - download "http://ec2-50-18-79-139.us-west-1.compute.amazonaws.com/data/spark-1.2.1-SNAPSHOT-bin-hadoop2-without-hive.tgz" "spark" + download "http://d3jw87u4immizc.cloudfront.net/spark-tarball/spark-${spark.version}-bin-hadoop2-without-hive.tgz" "spark" cp -f $HIVE_ROOT/data/conf/spark/log4j.properties $BASE_DIR/spark/conf/ diff --git a/itests/qtest-spark/pom.xml b/itests/qtest-spark/pom.xml index ac34328..d12fad5 100644 --- a/itests/qtest-spark/pom.xml +++ b/itests/qtest-spark/pom.xml @@ -54,38 +54,6 @@ test - org.apache.tez - tez-api - ${tez.version} - true - - - org.apache.hadoop - hadoop-common - - - org.apache.hadoop - hadoop-mapreduce-client-core - - - org.apache.hadoop - hadoop-mapreduce-client-jobclient - - - org.apache.hadoop - hadoop-mapreduce-client-common - - - org.apache.hadoop - hadoop-hdfs - - - org.apache.hadoop - hadoop-yarn-client - - - - org.eclipse.jetty jetty-util ${spark.jetty.version} diff --git a/pom.xml b/pom.xml index dbb6bad..53e3f0a 100644 --- a/pom.xml +++ b/pom.xml @@ -154,7 +154,7 @@ 4.0.4 0.5.2 2.2.0 - 1.2.1-SNAPSHOT + 1.2.0 2.10 2.10.4 1.1 @@ -833,6 +833,7 @@ en_US.UTF-8 ${test.tmp.dir}/conf:${basedir}/${hive.path.to.root}/conf ${test.hive.hadoop.classpath} + ${test.hive.hadoop.classpath} ${env.PATH}${test.extra.path} diff --git a/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientImpl.java b/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientImpl.java index 54eacfd..8c8b592 100644 --- a/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientImpl.java +++ b/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientImpl.java @@ -38,6 +38,7 @@ import com.google.common.base.Charsets; import com.google.common.base.Joiner; +import com.google.common.base.Strings; import com.google.common.base.Preconditions; import com.google.common.base.Throwables; import com.google.common.collect.Lists; @@ -63,6 +64,7 @@ private static final String DRIVER_OPTS_KEY = "spark.driver.extraJavaOptions"; private static final String EXECUTOR_OPTS_KEY = "spark.executor.extraJavaOptions"; + private static final String DRIVER_EXTRA_CLASSPATH = "spark.driver.extraClassPath"; private final Map conf; private final AtomicInteger childIdGenerator; @@ -226,6 +228,17 @@ public void run() { allProps.put(DRIVER_OPTS_KEY, driverJavaOpts); allProps.put(EXECUTOR_OPTS_KEY, executorJavaOpts); + String hiveHadoopTestClasspath = Strings.nullToEmpty(System.getenv("HIVE_HADOOP_TEST_CLASSPATH")); + if (!hiveHadoopTestClasspath.isEmpty()) { + String extraClasspath = Strings.nullToEmpty((String)allProps.get(DRIVER_EXTRA_CLASSPATH)); + if (extraClasspath.isEmpty()) { + allProps.put(DRIVER_EXTRA_CLASSPATH, hiveHadoopTestClasspath); + } else { + extraClasspath = extraClasspath.endsWith(File.pathSeparator) ? extraClasspath : extraClasspath + File.pathSeparator; + allProps.put(DRIVER_EXTRA_CLASSPATH, extraClasspath + hiveHadoopTestClasspath); + } + } + Writer writer = new OutputStreamWriter(new FileOutputStream(properties), Charsets.UTF_8); try { allProps.store(writer, "Spark Context configuration");