diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/HiveEnvUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/HiveEnvUtils.java new file mode 100644 index 0000000..ca52224 --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/HiveEnvUtils.java @@ -0,0 +1,154 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.exec.mr; + +import java.io.File; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.hive.conf.HiveConf; + +import com.google.common.collect.Interner; +import com.google.common.collect.Interners; + +/** + * Hive environment utils. + */ +public class HiveEnvUtils { + public enum OSType { + OS_TYPE_LINUX, + OS_TYPE_WIN, + OS_TYPE_SOLARIS, + OS_TYPE_MAC, + OS_TYPE_FREEBSD, + OS_TYPE_OTHER + } + + public static final OSType osType = getOSType(); + + static private OSType getOSType() { + String osName = System.getProperty("os.name"); + if (osName.startsWith("Windows")) { + return OSType.OS_TYPE_WIN; + } else if (osName.contains("SunOS") || osName.contains("Solaris")) { + return OSType.OS_TYPE_SOLARIS; + } else if (osName.contains("Mac")) { + return OSType.OS_TYPE_MAC; + } else if (osName.contains("FreeBSD")) { + return OSType.OS_TYPE_FREEBSD; + } else if (osName.startsWith("Linux")) { + return OSType.OS_TYPE_LINUX; + } else { + // Some other form of Unix + return OSType.OS_TYPE_OTHER; + } + } + + // Helper static vars for each platform + public static final boolean WINDOWS = (osType == OSType.OS_TYPE_WIN); + + public static final String ENV_NAME_REGEX = "[A-Za-z_][A-Za-z0-9_]*"; + private static final Pattern VAR_SUBBER = + Pattern.compile((WINDOWS) + ? "%(" + ENV_NAME_REGEX + "?)%" + : "\\$(" + ENV_NAME_REGEX + ")"); + private static final Pattern VARVAL_SPLITTER = Pattern.compile( + "(?<=^|,)" // preceded by ',' or line begin + + "([A-Za-z_][A-Za-z0-9_]*)" // var group + + '=' + + "([^,]*)" // val group + ); + + /** + * Retains a weak reference to each string instance it has interned. + */ + private final static Interner weakInterner; + + static { + weakInterner = Interners.newWeakInterner(); + } + + public static void setEnvFromInputString(Map env, + String envString, HiveConf conf) { + String classPathSeparator = + conf.getBoolean("mapreduce.app-submission.cross-platform", false) + ? "" : File.pathSeparator; + setEnvFromInputString(env, envString, classPathSeparator); + } + + public static void setEnvFromInputString(Map env, + String envString, String classPathSeparator) { + if (envString != null && envString.length() > 0) { + Matcher varValMatcher = VARVAL_SPLITTER.matcher(envString); + while (varValMatcher.find()) { + String envVar = varValMatcher.group(1); + Matcher m = VAR_SUBBER.matcher(varValMatcher.group(2)); + StringBuffer sb = new StringBuffer(); + while (m.find()) { + String var = m.group(1); + // replace $env with the child's env constructed by tt's + String replace = env.get(var); + // if this key is not configured by the tt for the child .. get it + // from the tt's env + if (replace == null) + replace = System.getenv(var); + // the env key is note present anywhere .. simply set it + if (replace == null) + replace = ""; + m.appendReplacement(sb, Matcher.quoteReplacement(replace)); + } + m.appendTail(sb); + addToEnvironment(env, envVar, sb.toString(), classPathSeparator); + } + } + } + + @Public + @Unstable + public static void addToEnvironment( + Map environment, + String variable, String value, String classPathSeparator) { + String val = environment.get(variable); + if (val == null) { + val = value; + } else { + val = val + classPathSeparator + value; + } + environment.put(weakIntern(variable), weakIntern(val)); + } + + /** + * Interns and returns a reference to the representative instance + * for any of a collection of string instances that are equal to each other. + * Retains weak reference to the instance, + * and so does not prevent it from being garbage-collected. + * + * @param sample string instance to be interned + * @return weak reference to interned string instance + */ + public static String weakIntern(String sample) { + if (sample == null) { + return null; + } + return weakInterner.intern(sample); + } +} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapredLocalTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapredLocalTask.java index a5c1463..e20e9f6 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapredLocalTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapredLocalTask.java @@ -264,6 +264,14 @@ public int executeInChildVM(DriverContext driverContext) { MapRedTask.configureDebugVariablesForChildJVM(variables); } + // Set local map env + String mapEnv = + conf.get("mapreduce.map.env", conf.get("mapred.child.env")); + if ((getTaskTag() == HINTED_MAPJOIN_LOCAL || + getTaskTag() == CONVERTED_MAPJOIN_LOCAL) && + mapEnv != null) { + HiveEnvUtils.setEnvFromInputString(variables, mapEnv, conf); + } if(UserGroupInformation.isSecurityEnabled() && UserGroupInformation.isLoginKeytabBased()) {