Index: shims/ivy.xml =================================================================== --- shims/ivy.xml +++ shims/ivy.xml @@ -15,7 +15,7 @@ See the License for the specific language governing permissions and limitations under the License. --> - + @@ -27,10 +27,7 @@ - - - - + @@ -45,6 +42,84 @@ transitive="false"/> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Index: shims/build.xml =================================================================== --- shims/build.xml +++ shims/build.xml @@ -22,35 +22,9 @@ Before you can run these subtargets directly, you need to call at top-level: ant deploy-contrib compile-core-test --> - + - - - - - - - - - - - - - - - - - - - - - - - - - - @@ -62,11 +36,37 @@ - - + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - - - - Building shims @{shimName} - - - - - - + + + + Building shims @{shimName} + + + + + + + + - @@ -111,7 +113,7 @@ deprecation="${javac.deprecation}" includeantruntime="false"> - + - + - + Index: builtins/ivy.xml =================================================================== --- builtins/ivy.xml +++ builtins/ivy.xml @@ -23,6 +23,14 @@ https://cwiki.apache.org/confluence/display/Hive/Home - - + + + + + + + + Index: builtins/build.xml =================================================================== --- builtins/build.xml +++ builtins/build.xml @@ -17,12 +17,12 @@ limitations under the License. --> - + - + @@ -32,15 +32,18 @@ - + + - + Index: build.properties =================================================================== --- build.properties +++ build.properties @@ -26,21 +26,26 @@ javac.args= javac.args.warnings= -hadoop-0.20.version=0.20.1 +hadoop-0.20.version=0.20.2 hadoop-0.20S.version=1.0.0 -hadoop-0.23.version=0.23.0 +hadoop-0.23.version=0.23.1 hadoop.version=${hadoop-0.20.version} hadoop.security.version=${hadoop-0.20S.version} hadoop.mirror=http://mirror.facebook.net/facebook/hive-deps hadoop.mirror2=http://archive.cloudera.com/hive-deps +# Used to determine which set of Hadoop artifacts we depend on. +# - 20: hadoop-core, hadoop-test +# - 23: hadoop-common, hadoop-mapreduce-*, etc +hadoop.mr.rev=20 build.dir.hive=${hive.root}/build build.dir.hadoop=${build.dir.hive}/hadoopcore hadoop.version.ant-internal=${hadoop.version} hadoop.root.default=${build.dir.hadoop}/hadoop-${hadoop.version.ant-internal} hadoop.root=${hadoop.root.default} +test.hadoop.bin.path=${hive.root}/testutils/hadoop # Newer versions of Hadoop name the jar as hadoop-{core,test}-VERSION instead of hadoop-VERSION-{core,test} # We will add both styles to the classpath and it will pick up whichever is there hadoop.oldstyle-name.jar=${hadoop.root}/hadoop-${hadoop.version.ant-internal}-core.jar @@ -76,6 +81,9 @@ ivy.conf.dir=${hive.root}/ivy ivy.version=2.1.0 ivy.jar=${build.ivy.lib.dir}/ivy-${ivy.version}.jar +ivy.changingPattern=.*SNAPSHOT +ivy.publish.pattern=[artifact]-[revision].[ext] +ivy.artifact.retrieve.pattern=[conf]/[artifact]-[revision](-[classifier]).[ext] ivysettings.xml=${ivy.conf.dir}/ivysettings.xml ivyresolvelog=download-only ivy.mvn.repo=http://repo2.maven.org/maven2 Index: hbase-handler/ivy.xml =================================================================== --- hbase-handler/ivy.xml +++ hbase-handler/ivy.xml @@ -26,17 +26,8 @@ - - - - - - + conf="compile->default" /> - - - - - - - - - - - - - - - - - - - - - - - - - Index: build.xml =================================================================== --- build.xml +++ build.xml @@ -501,6 +501,7 @@ + @@ -686,13 +687,6 @@ - - - - - - - Index: testutils/hadoop =================================================================== --- /dev/null +++ testutils/hadoop @@ -0,0 +1,109 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script satisfies Hive's bin/hadoop dependency when running tests. +# It's a quick and dirty hack, and should eventually be made obsolete by +# completely eliminating Hive's dependency on bin/hadoop. + +bin=`which $0` +bin=`dirname ${bin}` +bin=`cd "$bin"; pwd` + +# Newer versions of glibc use an arena memory allocator that causes virtual +# memory usage to explode. This interacts badly with the many threads that +# we use in Hadoop. Tune the variable down to prevent vmem explosion. +export MALLOC_ARENA_MAX=${MALLOC_ARENA_MAX:-4} + +JAVA=$JAVA_HOME/bin/java +JAVA_HEAP_MAX=-Xmx1000m + +# check envvars which might override default args +if [ "$HADOOP_HEAPSIZE" != "" ]; then + JAVA_HEAP_MAX="-Xmx""$HADOOP_HEAPSIZE""m" +fi + +# CLASSPATH initially contains $HADOOP_CONF_DIR +CLASSPATH="${HADOOP_CONF_DIR}" + +# so that filenames w/ spaces are handled correctly in loops below +IFS= + +BUILD_ROOT="$bin"/../build + +if [ "$HIVE_HADOOP_TEST_CLASSPATH" != "" ]; then + CLASSPATH=${CLASSPATH}:${HIVE_HADOOP_TEST_CLASSPATH} +else + echo "Error: HIVE_HADOOP_TEST_CLASSPATH not defined." + exit 1 +fi + +# restore ordinary behaviour +unset IFS + +mkdir -p $BUILD_ROOT/test/hadoop/logs +HADOOP_LOG_DIR=$BUILD_ROOT/test/hadoop/logs +HADOOP_LOGFILE=hadoop.log + +HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.log.dir=$HADOOP_LOG_DIR" +HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.log.file=$HADOOP_LOGFILE" +HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.home.dir=$HADOOP_PREFIX" +HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.id.str=$HADOOP_IDENT_STRING" +HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.root.logger=${HADOOP_ROOT_LOGGER:-INFO,console}" +if [ "x$JAVA_LIBRARY_PATH" != "x" ]; then + HADOOP_OPTS="$HADOOP_OPTS -Djava.library.path=$JAVA_LIBRARY_PATH" +fi +HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.policy.file=$HADOOP_POLICYFILE" + +# Disable ipv6 as it can cause issues +HADOOP_OPTS="$HADOOP_OPTS -Djava.net.preferIPv4Stack=true" + +COMMAND=$1 +# the core commands +if [ "$COMMAND" = "fs" ] ; then + CLASS=org.apache.hadoop.fs.FsShell + HADOOP_OPTS="$HADOOP_OPTS $HADOOP_CLIENT_OPTS" +elif [ "$COMMAND" = "version" ] ; then + CLASS=org.apache.hadoop.util.VersionInfo + HADOOP_OPTS="$HADOOP_OPTS $HADOOP_CLIENT_OPTS" +elif [ "$COMMAND" = "jar" ] ; then + CLASS=org.apache.hadoop.util.RunJar +elif [ "$COMMAND" = "distcp" ] ; then + CLASS=org.apache.hadoop.tools.DistCp + CLASSPATH=${CLASSPATH}:${TOOL_PATH} + HADOOP_OPTS="$HADOOP_OPTS $HADOOP_CLIENT_OPTS" +elif [ "$COMMAND" = "daemonlog" ] ; then + CLASS=org.apache.hadoop.log.LogLevel + HADOOP_OPTS="$HADOOP_OPTS $HADOOP_CLIENT_OPTS" +elif [ "$COMMAND" = "archive" ] ; then + CLASS=org.apache.hadoop.tools.HadoopArchives + CLASSPATH=${CLASSPATH}:${TOOL_PATH} + HADOOP_OPTS="$HADOOP_OPTS $HADOOP_CLIENT_OPTS" +elif [[ "$COMMAND" = -* ]] ; then + # class and package names cannot begin with a - + echo "Error: No command named \`$COMMAND' was found. Perhaps you meant \`hadoop ${COMMAND#-}'" + exit 1 +else + HADOOP_OPTS="$HADOOP_OPTS $HADOOP_CLIENT_OPTS" + CLASS=$COMMAND +fi +shift + +#make sure security appender is turned off +HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.security.logger=${HADOOP_SECURITY_LOGGER:-INFO,NullAppender}" + +export CLASSPATH=$CLASSPATH +exec "$JAVA" $JAVA_HEAP_MAX $HADOOP_OPTS $CLASS "$@" Index: jdbc/ivy.xml =================================================================== --- jdbc/ivy.xml +++ jdbc/ivy.xml @@ -27,14 +27,8 @@ - - - + conf="compile->default" /> - - @@ -37,15 +30,8 @@ - - - - - - - - - + + Index: metastore/ivy.xml =================================================================== --- metastore/ivy.xml +++ metastore/ivy.xml @@ -26,15 +26,10 @@ - - - - + conf="compile->default" /> + conf="compile->default" transitive="false" /> @@ -62,10 +57,10 @@ - - + + + + Index: ivy/common-configurations.xml =================================================================== --- ivy/common-configurations.xml +++ ivy/common-configurations.xml @@ -18,6 +18,14 @@ - + + + + + + + + + \ No newline at end of file Index: ivy/ivysettings.xml =================================================================== --- ivy/ivysettings.xml +++ ivy/ivysettings.xml @@ -36,6 +36,8 @@ + + @@ -62,16 +64,18 @@ + + @@ -86,7 +90,8 @@ - + + + + + - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + @@ -125,47 +143,41 @@ - - - + - - - + - + - + - + - - - - - - + + + + + + - - - - - - - - - @@ -178,10 +190,10 @@ - - + + + @@ -251,10 +263,13 @@ + - + + + + + + + + + + + - - + - + @@ -391,8 +415,9 @@ + - + - - - - - - + conf="compile->default" /> + + + Index: hwi/build.xml =================================================================== --- hwi/build.xml +++ hwi/build.xml @@ -37,18 +37,6 @@ - - - - - - - - - - - - @@ -78,7 +66,7 @@ debug="${javac.debug}" deprecation="${javac.deprecation}"> - + Index: common/ivy.xml =================================================================== --- common/ivy.xml +++ common/ivy.xml @@ -27,11 +27,62 @@ - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + conf="compile->default" transitive="false" /> - - + conf="compile->default" /> - - - - - - - - - - - - - - - - - You must set the 'thrift.home' property! Index: contrib/ivy.xml =================================================================== --- contrib/ivy.xml +++ contrib/ivy.xml @@ -26,19 +26,10 @@ - - - - - - + conf="compile->default" /> + conf="compile->default" transitive="false" /> - - - - - - - - - - - - - - - - - - - - - - - - - - Index: serde/ivy.xml =================================================================== --- serde/ivy.xml +++ serde/ivy.xml @@ -26,13 +26,10 @@ - - - + conf="compile->default" /> + conf="compile->default" transitive="false" /> Index: cli/ivy.xml =================================================================== --- cli/ivy.xml +++ cli/ivy.xml @@ -28,9 +28,7 @@ - - - + - - - - + conf="compile->default" /> - - - - - + conf="compile->default" /> + conf="test->default,runtime" transitive="false"/> + + + + + + + + + + + + + + + + + Index: ql/build.xml =================================================================== --- ql/build.xml +++ ql/build.xml @@ -18,7 +18,7 @@ --> - + @@ -32,37 +32,6 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - You must set the 'thrift.home' property! @@ -245,6 +214,9 @@ + Index: pdk/ivy.xml =================================================================== --- pdk/ivy.xml +++ pdk/ivy.xml @@ -27,5 +27,7 @@ + Index: pdk/scripts/build-plugin.xml =================================================================== --- pdk/scripts/build-plugin.xml +++ pdk/scripts/build-plugin.xml @@ -33,19 +33,21 @@ - - - - - + + + + + - - + + + + @@ -118,12 +120,18 @@ + + + + + + Index: pdk/build.xml =================================================================== --- pdk/build.xml +++ pdk/build.xml @@ -52,11 +52,14 @@ target="package" inheritAll="false" inheritRefs="false"> + + +