diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/ScriptOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/ScriptOperator.java index f2eed44..465c508 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/ScriptOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/ScriptOperator.java @@ -28,6 +28,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; +import java.util.Collections; import java.util.HashSet; import java.util.Iterator; import java.util.Map; @@ -146,16 +147,19 @@ String safeEnvVarValue(String value, String name, boolean truncate) { return value; } - boolean blackListed(String name) { + /** + * Checks whether a given configuration name is blacklisted and should not be converted + * to an environment variable. + */ + boolean blackListed(Configuration conf, String name) { if (blackListedConfEntries == null) { blackListedConfEntries = new HashSet(); - if (hconf != null) { - String bl = hconf.get(HiveConf.ConfVars.HIVESCRIPT_ENV_BLACKLIST.toString()); - if (bl != null && bl.length() > 0) { + if (conf != null) { + String bl = conf.get(HiveConf.ConfVars.HIVESCRIPT_ENV_BLACKLIST.toString(), + HiveConf.ConfVars.HIVESCRIPT_ENV_BLACKLIST.getDefaultValue()); + if (bl != null && !bl.isEmpty()) { String[] bls = bl.split(","); - for (String b : bls) { - blackListedConfEntries.add(b); - } + Collections.addAll(blackListedConfEntries, bls); } } } @@ -171,7 +175,7 @@ void addJobConfToEnvironment(Configuration conf, Map env) { while (it.hasNext()) { Map.Entry en = it.next(); String name = en.getKey(); - if (!blackListed(name)) { + if (!blackListed(conf, name)) { // String value = (String)en.getValue(); // does not apply variable // expansion String value = conf.get(name); // does variable expansion @@ -306,8 +310,7 @@ boolean allowPartialConsumption() { void displayBrokenPipeInfo() { if (isLogInfoEnabled) { LOG.info("The script did not consume all input data. This is considered as an error."); - LOG.info("set " + HiveConf.ConfVars.ALLOWPARTIALCONSUMP.toString() - + "=true; to ignore it."); + LOG.info("set " + HiveConf.ConfVars.ALLOWPARTIALCONSUMP.toString() + "=true; to ignore it."); } return; } @@ -349,12 +352,12 @@ public void process(Object row, int tag) throws HiveException { } String[] wrappedCmdArgs = addWrapper(cmdArgs); - if (isLogInfoEnabled) { - LOG.info("Executing " + Arrays.asList(wrappedCmdArgs)); - LOG.info("tablename=" + tableName); - LOG.info("partname=" + partitionName); - LOG.info("alias=" + alias); - } + if (isLogInfoEnabled) { + LOG.info("Executing " + Arrays.asList(wrappedCmdArgs)); + LOG.info("tablename=" + tableName); + LOG.info("partname=" + partitionName); + LOG.info("alias=" + alias); + } ProcessBuilder pb = new ProcessBuilder(wrappedCmdArgs); Map env = pb.environment(); @@ -672,9 +675,9 @@ public void processLine(Writable line) throws HiveException { long now = System.currentTimeMillis(); // reporter is a member variable of the Operator class. if (now - lastReportTime > 60 * 1000 && reporter != null) { - if (isLogInfoEnabled) { - LOG.info("ErrorStreamProcessor calling reporter.progress()"); - } + if (isLogInfoEnabled) { + LOG.info("ErrorStreamProcessor calling reporter.progress()"); + } lastReportTime = now; reporter.progress(); } @@ -730,9 +733,9 @@ public void run() { } proc.processLine(row); } - if (isLogInfoEnabled) { - LOG.info("StreamThread " + name + " done"); - } + if (isLogInfoEnabled) { + LOG.info("StreamThread " + name + " done"); + } } catch (Throwable th) { scriptError = th; diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/TestOperators.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/TestOperators.java index 62057d8..bf122e0 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/exec/TestOperators.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/TestOperators.java @@ -57,6 +57,7 @@ import org.apache.hadoop.mapred.InputSplit; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.TextInputFormat; +import org.junit.Assert; import org.junit.Test; /** @@ -191,6 +192,21 @@ public void testScriptOperatorEnvVarsProcessing() throws Throwable { } } + public void testScriptOperatorBlacklistedEnvVarsProcessing() { + ScriptOperator scriptOperator = new ScriptOperator(); + + Configuration hconf = new JobConf(ScriptOperator.class); + + Map env = new HashMap(); + + HiveConf.setVar(hconf, HiveConf.ConfVars.HIVESCRIPT_ENV_BLACKLIST, "foobar"); + hconf.set("foobar", "foobar"); + hconf.set("barfoo", "barfoo"); + scriptOperator.addJobConfToEnvironment(hconf, env); + Assert.assertFalse(env.containsKey("foobar")); + Assert.assertTrue(env.containsKey("barfoo")); + } + public void testScriptOperator() throws Throwable { try { System.out.println("Testing Script Operator");