diff --git a/itests/bin/download-spark.sh b/itests/bin/download-spark.sh new file mode 100644 index 0000000000..ab2c9a56e0 --- /dev/null +++ b/itests/bin/download-spark.sh @@ -0,0 +1,41 @@ +download() { + url=$1; + finalName=$2 + tarName=$(basename $url) + rm -rf $BASE_DIR/$finalName + if [[ ! -f $DOWNLOAD_DIR/$tarName ]] + then + curl -Sso $DOWNLOAD_DIR/$tarName $url + else + local md5File="$tarName".md5sum + curl -Sso $DOWNLOAD_DIR/$md5File "$url".md5sum + cd $DOWNLOAD_DIR + if type md5sum >/dev/null 2>&1 && ! md5sum -c $md5File; then + curl -Sso $DOWNLOAD_DIR/$tarName $url || return 1 + elif type md5 >/dev/null 2>&1; then + md5chksum=$(md5 -r $tarName) + if [[ $(< $md5File) != $md5chksum ]]; then + curl -Sso $DOWNLOAD_DIR/$tarName $url || return 1 + fi + fi + + cd - + fi + + tar -zxf $DOWNLOAD_DIR/$tarName -C $BASE_DIR + mv $BASE_DIR/spark-$SPARK_VERSION-bin-hadoop3-beta1-without-hive $BASE_DIR/$finalName +} + +set -x +/bin/pwd +cd ../.. +[ ! -d ql ] && echo "expected to be at hive root?" && exit 1 + +BASE_DIR=itests/target +DOWNLOAD_DIR=itests/thirdparty +SPARK_VERSION=$1 + +mkdir -p $DOWNLOAD_DIR +download "http://d3jw87u4immizc.cloudfront.net/spark-tarball/spark-$SPARK_VERSION-bin-hadoop3-beta1-without-hive.tgz" "spark" +cp -f data/conf/spark/log4j2.properties $BASE_DIR/spark/conf/ + diff --git a/itests/hive-unit/pom.xml b/itests/hive-unit/pom.xml index 711856e3c6..6c83217a6e 100644 --- a/itests/hive-unit/pom.xml +++ b/itests/hive-unit/pom.xml @@ -543,7 +543,7 @@ - + diff --git a/itests/pom.xml b/itests/pom.xml index 9df73796d8..345e9220df 100644 --- a/itests/pom.xml +++ b/itests/pom.xml @@ -53,65 +53,13 @@ spark-test - + !skipSparkTests - + qtest-spark - - - - org.apache.maven.plugins - maven-antrun-plugin - - - download-spark - generate-sources - - run - - - - - set -x - /bin/pwd - BASE_DIR=./target - HIVE_ROOT=$BASE_DIR/../../../ - DOWNLOAD_DIR=./../thirdparty - download() { - url=$1; - finalName=$2 - tarName=$(basename $url) - rm -rf $BASE_DIR/$finalName - if [[ ! -f $DOWNLOAD_DIR/$tarName ]] - then - curl -Sso $DOWNLOAD_DIR/$tarName $url - else - local md5File="$tarName".md5sum - curl -Sso $DOWNLOAD_DIR/$md5File "$url".md5sum - cd $DOWNLOAD_DIR - if type md5sum >/dev/null && ! md5sum -c $md5File; then - curl -Sso $DOWNLOAD_DIR/$tarName $url || return 1 - fi - - cd - - fi - tar -zxf $DOWNLOAD_DIR/$tarName -C $BASE_DIR - mv $BASE_DIR/spark-${spark.version}-bin-hadoop3-beta1-without-hive $BASE_DIR/$finalName - } - mkdir -p $DOWNLOAD_DIR - download "http://d3jw87u4immizc.cloudfront.net/spark-tarball/spark-${spark.version}-bin-hadoop3-beta1-without-hive.tgz" "spark" - cp -f $HIVE_ROOT/data/conf/spark/log4j2.properties $BASE_DIR/spark/conf/ - - - - - - - - diff --git a/itests/qtest-spark/pom.xml b/itests/qtest-spark/pom.xml index d77dc89afc..d7dff93834 100644 --- a/itests/qtest-spark/pom.xml +++ b/itests/qtest-spark/pom.xml @@ -398,7 +398,7 @@ - +