diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 2723dadd76418f23b6e6358d21304e3312ef1cbd..ef5cca559be67463dbbf8b4fe16576ca61f76079 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -40,12 +40,11 @@ import org.slf4j.LoggerFactory; import javax.security.auth.login.LoginException; -import java.io.ByteArrayOutputStream; -import java.io.File; -import java.io.IOException; -import java.io.InputStream; -import java.io.PrintStream; + +import java.io.*; import java.net.URL; +import java.net.URLDecoder; +import java.net.URLEncoder; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -96,6 +95,35 @@ public void setSparkConfigUpdated(boolean isSparkConfigUpdated) { this.isSparkConfigUpdated = isSparkConfigUpdated; } + public interface EncoderDecoder { + V encode(K key); + K decode(V value); + } + + public static class URLEncoderDecoder implements EncoderDecoder { + private static final String UTF_8 = "UTF-8"; + @Override + public String encode(String key) { + try { + return URLEncoder.encode(key, UTF_8); + } catch (UnsupportedEncodingException e) { + return key; + } + } + + @Override + public String decode(String value) { + try { + return URLDecoder.decode(value, UTF_8); + } catch (UnsupportedEncodingException e) { + return value; + } + } + } + public static class EncoderDecoderFactory { + public static final URLEncoderDecoder URL_ENCODER_DECODER = new URLEncoderDecoder(); + } + static { ClassLoader classLoader = Thread.currentThread().getContextClassLoader(); if (classLoader == null) { @@ -3280,10 +3308,8 @@ public void setBoolVar(ConfVars var, boolean val) { public static String getVar(Configuration conf, ConfVars var) { assert (var.valClass == String.class) : var.varname; - if (var.altName != null) { - return conf.get(var.varname, conf.get(var.altName, var.defaultStrVal)); - } - return conf.get(var.varname, var.defaultStrVal); + return var.altName != null ? conf.get(var.varname, conf.get(var.altName, var.defaultStrVal)) + : conf.get(var.varname, var.defaultStrVal); } public static String getTrimmedVar(Configuration conf, ConfVars var) { @@ -3306,10 +3332,13 @@ public static String getTrimmedVar(Configuration conf, ConfVars var) { } public static String getVar(Configuration conf, ConfVars var, String defaultVal) { - if (var.altName != null) { - return conf.get(var.varname, conf.get(var.altName, defaultVal)); - } - return conf.get(var.varname, defaultVal); + String ret = var.altName != null ? conf.get(var.varname, conf.get(var.altName, defaultVal)) + : conf.get(var.varname, defaultVal); + return ret; + } + + public static String getVar(Configuration conf, ConfVars var, EncoderDecoder encoderDecoder) { + return encoderDecoder.decode(getVar(conf, var)); } public String getLogIdVar(String defaultValue) { @@ -3330,6 +3359,10 @@ public static void setVar(Configuration conf, ConfVars var, String val) { assert (var.valClass == String.class) : var.varname; conf.set(var.varname, val); } + public static void setVar(Configuration conf, ConfVars var, String val, + EncoderDecoder encoderDecoder) { + setVar(conf, var, encoderDecoder.encode(val)); + } public static ConfVars getConfVars(String name) { return vars.get(name); @@ -3347,6 +3380,21 @@ public void setVar(ConfVars var, String val) { setVar(this, var, val); } + public String getQueryString() { + return getQueryString(this); + } + + public static String getQueryString(Configuration conf) { + return getVar(conf, ConfVars.HIVEQUERYSTRING, EncoderDecoderFactory.URL_ENCODER_DECODER); + } + + public void setQueryString(String query) { + setQueryString(this, query); + } + + public static void setQueryString(Configuration conf, String query) { + setVar(conf, ConfVars.HIVEQUERYSTRING, query, EncoderDecoderFactory.URL_ENCODER_DECODER); + } public void logVars(PrintStream ps) { for (ConfVars one : ConfVars.values()) { ps.println(one.varname + "=" + ((get(one.varname) != null) ? get(one.varname) : "")); @@ -3901,7 +3949,7 @@ public static String checkCartesian(Configuration conf) { } private static boolean isAllowed(Configuration conf, ConfVars setting) { - String mode = HiveConf.getVar(conf, ConfVars.HIVEMAPREDMODE, null); + String mode = HiveConf.getVar(conf, ConfVars.HIVEMAPREDMODE, (String)null); return (mode != null) ? !"strict".equals(mode) : !HiveConf.getBoolVar(conf, setting); } } diff --git a/common/src/test/org/apache/hadoop/hive/conf/TestHiveConf.java b/common/src/test/org/apache/hadoop/hive/conf/TestHiveConf.java index 365d5005d2a1f240b12bb4e9f9a920b3db4d32ba..f88573f670eaa581485a6f13ebb21c776e8a7546 100644 --- a/common/src/test/org/apache/hadoop/hive/conf/TestHiveConf.java +++ b/common/src/test/org/apache/hadoop/hive/conf/TestHiveConf.java @@ -25,6 +25,8 @@ import org.junit.Assert; import org.junit.Test; +import java.io.UnsupportedEncodingException; +import java.net.URLEncoder; import java.util.concurrent.TimeUnit; @@ -156,4 +158,12 @@ public void testSparkConfigUpdate(){ conf.setSparkConfigUpdated(false); Assert.assertFalse(conf.getSparkConfigUpdated()); } + @Test + public void testEncodingDecoding() throws UnsupportedEncodingException { + HiveConf conf = new HiveConf(); + String query = "select blah, '\u0001' from random_table"; + conf.setQueryString(query); + Assert.assertEquals(URLEncoder.encode(query, "UTF-8"), conf.get(ConfVars.HIVEQUERYSTRING.varname)); + Assert.assertEquals(query, conf.getQueryString()); + } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java index 32531465cba95de6caa1bd292d489c1582efb942..b50c5a2c4e396975f3323b16426445f869ff7b89 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java @@ -86,7 +86,6 @@ import org.apache.hadoop.hive.ql.optimizer.ppr.PartitionPruner; import org.apache.hadoop.hive.ql.parse.ASTNode; import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer; -import org.apache.hadoop.hive.ql.parse.CalcitePlanner; import org.apache.hadoop.hive.ql.parse.ColumnAccessInfo; import org.apache.hadoop.hive.ql.parse.HiveSemanticAnalyzerHook; import org.apache.hadoop.hive.ql.parse.HiveSemanticAnalyzerHookContext; @@ -499,7 +498,7 @@ public void run() { plan = new QueryPlan(queryStr, sem, perfLogger.getStartTime(PerfLogger.DRIVER_RUN), queryId, SessionState.get().getHiveOperation(), schema); - conf.setVar(HiveConf.ConfVars.HIVEQUERYSTRING, queryStr); + conf.setQueryString(queryStr); conf.set("mapreduce.workflow.id", "hive_" + queryId); conf.set("mapreduce.workflow.name", queryStr); @@ -1484,7 +1483,7 @@ public int execute() throws CommandNeedRetryException { String queryId = plan.getQueryId(); // Get the query string from the conf file as the compileInternal() method might // hide sensitive information during query redaction. - String queryStr = HiveConf.getVar(conf, HiveConf.ConfVars.HIVEQUERYSTRING); + String queryStr = conf.getQueryString(); maxthreads = HiveConf.getIntVar(conf, HiveConf.ConfVars.EXECPARALLETHREADNUMBER); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/errors/TaskLogProcessor.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/errors/TaskLogProcessor.java index 68123d45c335e833984d3547607dc3ee7d8fd870..b788d8fd412416f090c1d8181fe5d7d014192a61 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/errors/TaskLogProcessor.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/errors/TaskLogProcessor.java @@ -53,7 +53,7 @@ private final String query; public TaskLogProcessor(JobConf conf) { - query = HiveConf.getVar(conf, HiveConf.ConfVars.HIVEQUERYSTRING); + query = HiveConf.getQueryString(conf); heuristics.put(new ScriptErrorHeuristic(), new HeuristicStats()); heuristics.put(new MapAggrMemErrorHeuristic(), new HeuristicStats()); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java index b184b4e554cf3f7a61932195b9b840cf13f3594b..ce020a52721b439c4a9a81ee75e9829f2da00221 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java @@ -568,7 +568,7 @@ protected void setInputAttributes(Configuration conf) { // Intentionally overwrites anything the user may have put here conf.setBoolean("hive.input.format.sorted", mWork.isInputFormatSorted()); - if (HiveConf.getVar(conf, ConfVars.HIVE_CURRENT_DATABASE, null) == null) { + if (HiveConf.getVar(conf, ConfVars.HIVE_CURRENT_DATABASE, (String)null) == null) { HiveConf.setVar(conf, ConfVars.HIVE_CURRENT_DATABASE, getCurrentDB()); } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java index 473dbd61bf538fe08953dcf7994d5821f6bdbacc..79da8602ab74d55287ed3360fce220012462eaf7 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java @@ -904,7 +904,7 @@ private void addTempResources(Configuration conf, public FileStatus getHiveJarDirectory(Configuration conf) throws IOException, LoginException { FileStatus fstatus = null; - String hdfsDirPathStr = HiveConf.getVar(conf, HiveConf.ConfVars.HIVE_JAR_DIRECTORY, null); + String hdfsDirPathStr = HiveConf.getVar(conf, HiveConf.ConfVars.HIVE_JAR_DIRECTORY, (String)null); if (hdfsDirPathStr != null) { LOG.info("Hive jar directory is " + hdfsDirPathStr); fstatus = validateTargetDir(new Path(hdfsDirPathStr), conf); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java index 70b2bc0cc15cd862cdb1adba0690d0530681ebde..4cbb907a26fb6170e25bdd166b59b9c1e30f9972 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java @@ -373,11 +373,11 @@ public SessionState(HiveConf conf, String userName) { } public void setCmd(String cmdString) { - conf.setVar(HiveConf.ConfVars.HIVEQUERYSTRING, cmdString); + conf.setQueryString(cmdString); } public String getCmd() { - return (conf.getVar(HiveConf.ConfVars.HIVEQUERYSTRING)); + return (conf.getQueryString()); } public String getQueryId() { @@ -1620,7 +1620,7 @@ public void setupQueryCurrentTimestamp() { // Provide a facility to set current timestamp during tests if (conf.getBoolVar(ConfVars.HIVE_IN_TEST)) { String overrideTimestampString = - HiveConf.getVar(conf, HiveConf.ConfVars.HIVETESTCURRENTTIMESTAMP, null); + HiveConf.getVar(conf, HiveConf.ConfVars.HIVETESTCURRENTTIMESTAMP, (String)null); if (overrideTimestampString != null && overrideTimestampString.length() > 0) { queryCurrentTimestamp = Timestamp.valueOf(overrideTimestampString); } diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/errors/TestTaskLogProcessor.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/errors/TestTaskLogProcessor.java index 67a86a60949a6b70f71593470a6d0ef8c265ee04..477479d6d3d61cf2794c4c563475393efc180535 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/exec/errors/TestTaskLogProcessor.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/errors/TestTaskLogProcessor.java @@ -98,7 +98,7 @@ private String writeThrowableAsFile(String before, Throwable t, String after, @Test public void testGetStackTraces() throws Exception { JobConf jobConf = new JobConf(); - jobConf.set(HiveConf.ConfVars.HIVEQUERYSTRING.varname, "select * from foo group by moo;"); + HiveConf.setQueryString(jobConf, "select * from foo group by moo;"); final TaskLogProcessor taskLogProcessor = new TaskLogProcessor(jobConf); @@ -150,7 +150,7 @@ private void checkException(String writenText, List actualTrace) throws @Test public void testScriptErrorHeuristic() throws Exception { JobConf jobConf = new JobConf(); - jobConf.set(HiveConf.ConfVars.HIVEQUERYSTRING.varname, "select * from foo group by moo;"); + HiveConf.setQueryString(jobConf, "select * from foo group by moo;"); final TaskLogProcessor taskLogProcessor = new TaskLogProcessor(jobConf); @@ -177,7 +177,7 @@ public void testScriptErrorHeuristic() throws Exception { @Test public void testDataCorruptErrorHeuristic() throws Exception { JobConf jobConf = new JobConf(); - jobConf.set(HiveConf.ConfVars.HIVEQUERYSTRING.varname, "select * from foo group by moo;"); + HiveConf.setQueryString(jobConf, "select * from foo group by moo;"); final TaskLogProcessor taskLogProcessor = new TaskLogProcessor(jobConf); @@ -210,7 +210,7 @@ public void testDataCorruptErrorHeuristic() throws Exception { @Test public void testMapAggrMemErrorHeuristic() throws Exception { JobConf jobConf = new JobConf(); - jobConf.set(HiveConf.ConfVars.HIVEQUERYSTRING.varname, "select * from foo group by moo;"); + HiveConf.setQueryString(jobConf, "select * from foo group by moo;"); final TaskLogProcessor taskLogProcessor = new TaskLogProcessor(jobConf); diff --git a/ql/src/test/org/apache/hadoop/hive/ql/hooks/TestHooks.java b/ql/src/test/org/apache/hadoop/hive/ql/hooks/TestHooks.java index 8d27762522f0ac5f59696e019ec52a25e77318db..4c14d8b83eb6c16a15d75994831d206f5677f7d1 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/hooks/TestHooks.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/hooks/TestHooks.java @@ -73,7 +73,7 @@ public void testQueryRedactor() throws Exception { Driver driver = createDriver(conf); int ret = driver.compile("select 'XXX' from t1"); assertEquals("Checking command success", 0, ret); - assertEquals("select 'AAA' from t1", HiveConf.getVar(conf, HiveConf.ConfVars.HIVEQUERYSTRING)); + assertEquals("select 'AAA' from t1", conf.getQueryString()); } public static class SimpleQueryRedactor extends Redactor {