diff --git a/bin/ext/beeline.sh b/bin/ext/beeline.sh index 90e562f..1872710 100644 --- a/bin/ext/beeline.sh +++ b/bin/ext/beeline.sh @@ -31,9 +31,15 @@ beeline () { hadoopClasspath="${HADOOP_CLASSPATH}:" fi export HADOOP_CLASSPATH="${hadoopClasspath}${HIVE_CONF_DIR}:${beelineJarPath}:${superCsvJarPath}:${jlineJarPath}:${jdbcStandaloneJarPath}" - export HADOOP_CLIENT_OPTS="$HADOOP_CLIENT_OPTS -Dlog4j.configurationFile=beeline-log4j2.properties " + BEELINE_LOG4J2_PROPERTIES="beeline-log4j2.properties" + export HADOOP_CLIENT_OPTS="${HADOOP_CLIENT_OPTS//${HIVE_LOG4J2_PROPERTIES}/${BEELINE_LOG4J2_PROPERTIES}}" - exec $HADOOP jar ${beelineJarPath} $CLASS $HIVE_OPTS "$@" + if [ "${USE_YARN_LAUNCHER}" = true ] ; then + export YARN_CLIENT_OPTS="${HADOOP_CLIENT_OPTS}" + export YARN_USER_CLASSPATH="${HADOOP_CLASSPATH}" + fi + + exec $JAR_LAUNCHER jar ${beelineJarPath} $CLASS $HIVE_OPTS "$@" } beeline_help () { diff --git a/bin/ext/hiveburninclient.sh b/bin/ext/hiveburninclient.sh index ccbe726..384b438 100644 --- a/bin/ext/hiveburninclient.sh +++ b/bin/ext/hiveburninclient.sh @@ -24,7 +24,7 @@ hiveburninclient() { HIVE_LIB=`cygpath -w "$HIVE_LIB"` fi JAR=${HIVE_LIB}/hive-service-*.jar - exec $HADOOP jar $JAR $CLASS $HIVE_OPTS "$@" + exec $JAR_LAUNCHER jar $JAR $CLASS $HIVE_OPTS "$@" } hiveburninclient_help() { diff --git a/bin/ext/hiveserver2.sh b/bin/ext/hiveserver2.sh index 1f02bb4..59f31bd 100644 --- a/bin/ext/hiveserver2.sh +++ b/bin/ext/hiveserver2.sh @@ -23,7 +23,7 @@ hiveserver2() { fi JAR=${HIVE_LIB}/hive-service-[0-9].*.jar - exec $HADOOP jar $JAR $CLASS $HIVE_OPTS "$@" + exec $JAR_LAUNCHER jar $JAR $CLASS $HIVE_OPTS "$@" } hiveserver2_help() { diff --git a/bin/ext/hplsql.sh b/bin/ext/hplsql.sh index ddaf324..4025e97 100644 --- a/bin/ext/hplsql.sh +++ b/bin/ext/hplsql.sh @@ -29,7 +29,11 @@ hplsql () { fi export HADOOP_CLASSPATH="${hadoopClasspath}${HIVE_CONF_DIR}:${hplsqlJarPath}:${antlrJarPath}" - exec $HADOOP jar ${hplsqlJarPath} $CLASS $HIVE_OPTS "$@" + if [ "${USE_YARN_LAUNCHER}" = true ] ; then + export YARN_USER_CLASSPATH="${HADOOP_CLASSPATH}" + fi + + exec $JAR_LAUNCHER jar ${hplsqlJarPath} $CLASS $HIVE_OPTS "$@" } hplsql_help () { diff --git a/bin/ext/hwi.sh b/bin/ext/hwi.sh index f9cd8ec..59efe55 100644 --- a/bin/ext/hwi.sh +++ b/bin/ext/hwi.sh @@ -41,8 +41,12 @@ hwi() { export HADOOP_CLASSPATH + if [ "${USE_YARN_LAUNCHER}" = true ] ; then + export YARN_USER_CLASSPATH="${HADOOP_CLASSPATH}" + fi + # hadoop 20 or newer - skip the aux_jars option and hiveconf - exec $HADOOP jar ${HWI_JAR_FILE} $CLASS $HIVE_OPTS "$@" + exec $JAR_LAUNCHER jar ${HWI_JAR_FILE} $CLASS $HIVE_OPTS "$@" } hwi_help(){ diff --git a/bin/ext/jar.sh b/bin/ext/jar.sh index b52f9a7..97c4ab8 100644 --- a/bin/ext/jar.sh +++ b/bin/ext/jar.sh @@ -38,7 +38,7 @@ jar () { fi # hadoop 20 or newer - skip the aux_jars option and hiveconf - exec $HADOOP jar $RUNJAR $RUNCLASS $HIVE_OPTS "$@" + exec $JAR_LAUNCHER jar $RUNJAR $RUNCLASS $HIVE_OPTS "$@" } jar_help () { diff --git a/bin/ext/lineage.sh b/bin/ext/lineage.sh index 993bc8d..5ef46cb 100644 --- a/bin/ext/lineage.sh +++ b/bin/ext/lineage.sh @@ -29,7 +29,7 @@ lineage () { HIVE_LIB=`cygpath -w "$HIVE_LIB"` fi - exec $HADOOP jar ${HIVE_LIB}/hive-exec-*.jar $CLASS "$@" + exec $JAR_LAUNCHER jar ${HIVE_LIB}/hive-exec-*.jar $CLASS "$@" } lineage_help () { diff --git a/bin/ext/metastore.sh b/bin/ext/metastore.sh index 095e6e2..21b54e9 100644 --- a/bin/ext/metastore.sh +++ b/bin/ext/metastore.sh @@ -27,7 +27,12 @@ metastore() { # hadoop 20 or newer - skip the aux_jars option and hiveconf export HADOOP_OPTS="$HIVE_METASTORE_HADOOP_OPTS $HADOOP_OPTS" - exec $HADOOP jar $JAR $CLASS "$@" + + if [ "${USE_YARN_LAUNCHER}" = true ] ; then + export YARN_OPTS="${HADOOP_OPTS}" + fi + + exec $JAR_LAUNCHER jar $JAR $CLASS "$@" } metastore_help() { diff --git a/bin/ext/util/execHiveCmd.sh b/bin/ext/util/execHiveCmd.sh index 9a06ce0..44f0dfe 100644 --- a/bin/ext/util/execHiveCmd.sh +++ b/bin/ext/util/execHiveCmd.sh @@ -43,5 +43,5 @@ execHiveCmd () { fi # hadoop 20 or newer - skip the aux_jars option. picked up from hiveconf - exec $HADOOP jar ${HIVE_LIB}/$JAR $CLASS $HIVE_OPTS "$@" + exec $JAR_LAUNCHER jar ${HIVE_LIB}/$JAR $CLASS $HIVE_OPTS "$@" } diff --git a/bin/ext/version.sh b/bin/ext/version.sh index b6a237b..8352861 100644 --- a/bin/ext/version.sh +++ b/bin/ext/version.sh @@ -27,7 +27,7 @@ version () { # hadoop 20 or newer - skip the aux_jars option and hiveconf CLASS=org.apache.hive.common.util.HiveVersionInfo - exec $HADOOP jar $JAR $CLASS 2>> ${STDERR} + exec $JAR_LAUNCHER jar $JAR $CLASS 2>> ${STDERR} } version_help () { diff --git a/bin/hive b/bin/hive index 3b9f3e0..6b51b9f 100755 --- a/bin/hive +++ b/bin/hive @@ -310,12 +310,30 @@ for j in $SERVICE_LIST ; do done # to initialize logging for all services -export HADOOP_CLIENT_OPTS="$HADOOP_CLIENT_OPTS -Dlog4j.configurationFile=hive-log4j2.properties " +HIVE_LOG4J2_PROPERTIES="hive-log4j2.properties" +export HADOOP_CLIENT_OPTS="$HADOOP_CLIENT_OPTS -Dlog4j.configurationFile=${HIVE_LOG4J2_PROPERTIES} " if [ -f "${HIVE_CONF_DIR}/parquet-logging.properties" ]; then export HADOOP_CLIENT_OPTS="$HADOOP_CLIENT_OPTS -Djava.util.logging.config.file=${HIVE_CONF_DIR}/parquet-logging.properties " else - export HADOOP_CLIENT_OPTS="$HADOOP_CLIENT_OPTS -Dlog4j.configurationFile=hive-log4j2.properties -Djava.util.logging.config.file=$bin/../conf/parquet-logging.properties " + export HADOOP_CLIENT_OPTS="$HADOOP_CLIENT_OPTS -Djava.util.logging.config.file=$bin/../conf/parquet-logging.properties " +fi + +# determine if services should use "hadoop jar" or "yarn jar" command +JAR_LAUNCHER=$HADOOP +USE_YARN_LAUNCHER=false +if [[ -n "${YARN_OPTS}" ]] || [[ -n "${YARN_CLIENT_OPTS}" ]]; then + YARN=${HADOOP_HOME}/bin/yarn + if [ -f ${YARN} ]; then + JAR_LAUNCHER=${YARN} + USE_YARN_LAUNCHER=true + # if we are using yarn command for launching service propagate HADOOP_OPTS and HADOOP_CLIENT_OPTS to it + export YARN_OPTS="$HADOOP_OPTS $YARN_OPTS" + export YARN_CLIENT_OPTS="$HADOOP_CLIENT_OPTS $YARN_CLIENT_OPTS" + export YARN_USER_CLASSPATH="$HADOOP_CLASSPATH" + # hive need this to be set to true for proper initialization of log4j2 and jline + export YARN_USER_CLASSPATH_FIRST="$HADOOP_USER_CLASSPATH_FIRST" + fi fi if [ "$TORUN" = "" ] ; then diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/HadoopJobExecHelper.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/HadoopJobExecHelper.java index 5656f9a..7c85eeb 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/HadoopJobExecHelper.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/HadoopJobExecHelper.java @@ -446,7 +446,7 @@ public void jobInfo(RunningJob rj) { } console.printInfo(getJobStartMsg(rj.getID()) + ", Tracking URL = " + rj.getTrackingURL()); - console.printInfo("Kill Command = " + HiveConf.getVar(job, HiveConf.ConfVars.HADOOPBIN) + console.printInfo("Kill Command = " + HiveConf.getVar(job, ConfVars.YARNBIN) + " job -kill " + rj.getID()); } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapRedTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapRedTask.java index ce1106d9..5b3f96d 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapRedTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapRedTask.java @@ -148,7 +148,7 @@ public int execute(DriverContext driverContext) { super.setInputAttributes(conf); // enable assertion - String hadoopExec = conf.getVar(HiveConf.ConfVars.HADOOPBIN); + String hadoopExec = conf.getVar(ConfVars.YARNBIN); String hiveJar = conf.getJar(); String libJars = super.getResource(conf, ResourceType.JAR); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapredLocalTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapredLocalTask.java index a26d2b8..413b457 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapredLocalTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapredLocalTask.java @@ -160,7 +160,7 @@ public int executeInChildVM(DriverContext driverContext) { Context ctx = driverContext.getCtx(); String hiveJar = conf.getJar(); - String hadoopExec = conf.getVar(HiveConf.ConfVars.HADOOPBIN); + String hadoopExec = conf.getVar(ConfVars.YARNBIN); conf.setVar(ConfVars.HIVEADDEDJARS, Utilities.getResourceFiles(conf, SessionState.ResourceType.JAR)); // write out the plan to a local file Path planPath = new Path(ctx.getLocalTmpPath(), "plan.xml");