diff --git cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java index a2976b5..d6e217f 100644 --- cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java +++ cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java @@ -581,6 +581,7 @@ public class CliDriver { } CliDriver cli = new CliDriver(); + cli.setHiveVariables(oproc.getHiveVariables()); // Execute -i init files (always in silent mode) cli.processInitFiles(ss); @@ -632,4 +633,8 @@ public class CliDriver { System.exit(ret); } + public void setHiveVariables(Map hiveVariables) { + SessionState.get().setHiveVariables(hiveVariables); + } + } diff --git cli/src/java/org/apache/hadoop/hive/cli/OptionsProcessor.java cli/src/java/org/apache/hadoop/hive/cli/OptionsProcessor.java index 90084ed..d9ec9d7 100644 --- cli/src/java/org/apache/hadoop/hive/cli/OptionsProcessor.java +++ cli/src/java/org/apache/hadoop/hive/cli/OptionsProcessor.java @@ -18,7 +18,9 @@ package org.apache.hadoop.hive.cli; +import java.util.HashMap; import java.util.Arrays; +import java.util.Map; import java.util.Properties; import org.apache.commons.cli.GnuParser; @@ -38,7 +40,7 @@ public class OptionsProcessor { protected static final Log l4j = LogFactory.getLog(OptionsProcessor.class.getName()); private final Options options = new Options(); private org.apache.commons.cli.CommandLine commandLine; - + Map hiveVariables = new HashMap(); @SuppressWarnings("static-access") public OptionsProcessor() { @@ -87,6 +89,24 @@ public class OptionsProcessor { .withDescription("connecting to Hive Server on port number") .create('p')); + // Substitution option -d, --define + options.addOption(OptionBuilder + .withValueSeparator() + .hasArgs(2) + .withArgName("key=value") + .withLongOpt("define") + .withDescription("Variable subsitution to apply to hive commands. e.g. -d A=B or --define A=B") + .create('d')); + + // Substitution option --hivevar + options.addOption(OptionBuilder + .withValueSeparator() + .hasArgs(2) + .withArgName("key=value") + .withLongOpt("hivevar") + .withDescription("Variable subsitution to apply to hive commands. e.g. --hivevar A=B") + .create()); + // [-S|--silent] options.addOption(new Option("S", "silent", false, "Silent mode in interactive shell")); @@ -105,6 +125,16 @@ public class OptionsProcessor { for (String propKey : confProps.stringPropertyNames()) { System.setProperty(propKey, confProps.getProperty(propKey)); } + + Properties hiveVars = commandLine.getOptionProperties("define"); + for (String propKey : hiveVars.stringPropertyNames()) { + hiveVariables.put(propKey, hiveVars.getProperty(propKey)); + } + + Properties hiveVars2 = commandLine.getOptionProperties("hivevar"); + for (String propKey : hiveVars2.stringPropertyNames()) { + hiveVariables.put(propKey, hiveVars2.getProperty(propKey)); + } } catch (ParseException e) { System.err.println(e.getMessage()); printUsage(); @@ -157,4 +187,8 @@ public class OptionsProcessor { private void printUsage() { new HelpFormatter().printHelp("hive", options); } + + public Map getHiveVariables() { + return hiveVariables; + } } diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/VariableSubstitution.java ql/src/java/org/apache/hadoop/hive/ql/parse/VariableSubstitution.java index e203dda..65c5e7d 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/VariableSubstitution.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/VariableSubstitution.java @@ -25,6 +25,7 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hadoop.hive.ql.processors.SetProcessor; +import org.apache.hadoop.hive.ql.session.SessionState; public class VariableSubstitution { @@ -32,6 +33,35 @@ public class VariableSubstitution { protected static Pattern varPat = Pattern.compile("\\$\\{[^\\}\\$\u0020]+\\}"); protected static int MAX_SUBST = 40; + private String getSubstitute(HiveConf conf, String var) { + String val = null; + try { + if (var.startsWith(SetProcessor.SYSTEM_PREFIX)) { + val = System.getProperty(var.substring(SetProcessor.SYSTEM_PREFIX.length())); + } + } catch(SecurityException se) { + l4j.warn("Unexpected SecurityException in Configuration", se); + } + if (val ==null){ + if (var.startsWith(SetProcessor.ENV_PREFIX)){ + val = System.getenv(var.substring(SetProcessor.ENV_PREFIX.length())); + } + } + if (val == null) { + if (var.startsWith(SetProcessor.HIVECONF_PREFIX)){ + val = conf.get(var.substring(SetProcessor.HIVECONF_PREFIX.length())); + } + } + if (val ==null){ + if(var.startsWith(SetProcessor.HIVEVAR_PREFIX)){ + val = SessionState.get().getHiveVariables().get(var.substring(SetProcessor.HIVEVAR_PREFIX.length())); + } else { + val = SessionState.get().getHiveVariables().get(var); + } + } + return val; + } + public String substitute (HiveConf conf, String expr) { if (conf.getBoolVar(ConfVars.HIVEVARIABLESUBSTITUTE)){ @@ -51,27 +81,11 @@ public class VariableSubstitution { } String var = match.group(); var = var.substring(2, var.length()-1); // remove ${ .. } - String val = null; - try { - if (var.startsWith(SetProcessor.SYSTEM_PREFIX)) { - val = System.getProperty(var.substring(SetProcessor.SYSTEM_PREFIX.length())); - } - } catch(SecurityException se) { - l4j.warn("Unexpected SecurityException in Configuration", se); - } - if (val ==null){ - if (var.startsWith(SetProcessor.ENV_PREFIX)){ - val = System.getenv(var.substring(SetProcessor.ENV_PREFIX.length())); - } - } - if (val == null) { - if (var.startsWith(SetProcessor.HIVECONF_PREFIX)){ - val = conf.get(var.substring(SetProcessor.HIVECONF_PREFIX.length())); - } - } + String val = getSubstitute(conf, var); + if (val == null) { l4j.debug("Interpolation result: "+eval); - return eval; // return literal ${var}: var is unbound + return eval; // return literal, no substitution found } // substitute eval = eval.substring(0, match.start())+val+eval.substring(match.end()); diff --git ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java index 97fa1ab..9ca5ca4 100644 --- ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java +++ ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java @@ -36,6 +36,7 @@ public class SetProcessor implements CommandProcessor { public static final String ENV_PREFIX = "env:"; public static final String SYSTEM_PREFIX = "system:"; public static final String HIVECONF_PREFIX = "hiveconf:"; + public static final String HIVEVAR_PREFIX = "hivevar:"; public static boolean getBoolean(String value) { if (value.equals("on") || value.equals("true")) { @@ -50,7 +51,6 @@ public class SetProcessor implements CommandProcessor { private void dumpOptions(Properties p) { SessionState ss = SessionState.get(); - SortedMap sortedMap = new TreeMap(); sortedMap.put("silent", (ss.getIsSilent() ? "on" : "off")); for (Object one : p.keySet()) { @@ -58,6 +58,12 @@ public class SetProcessor implements CommandProcessor { String oneValue = p.getProperty(oneProp); sortedMap.put(oneProp, oneValue); } + + // Inserting hive variables + for (String s : ss.getHiveVariables().keySet()) { + sortedMap.put(SetProcessor.HIVEVAR_PREFIX + s, ss.getHiveVariables().get(s)); + } + for (Map.Entry entries : sortedMap.entrySet()) { ss.out.println(entries.getKey() + "=" + entries.getValue()); } @@ -73,11 +79,13 @@ public class SetProcessor implements CommandProcessor { } - private void dumpOption(Properties p, String s) { + private void dumpOption(String s) { SessionState ss = SessionState.get(); - if (p.getProperty(s) != null) { - ss.out.println(s + "=" + p.getProperty(s)); + if (ss.getConf().get(s) != null) { + ss.out.println(s + "=" + ss.getConf().get(s)); + } else if (ss.getHiveVariables().containsKey(s)) { + ss.out.println(s + "=" + ss.getHiveVariables().get(s)); } else { ss.out.println(s + " is undefined"); } @@ -99,6 +107,10 @@ public class SetProcessor implements CommandProcessor { String propName = varname.substring(SetProcessor.HIVECONF_PREFIX.length()); ss.getConf().set(propName, new VariableSubstitution().substitute(ss.getConf(),varvalue)); return new CommandProcessorResponse(0); + } else if (varname.startsWith(SetProcessor.HIVEVAR_PREFIX)) { + String propName = varname.substring(SetProcessor.HIVEVAR_PREFIX.length()); + ss.getHiveVariables().put(propName, new VariableSubstitution().substitute(ss.getConf(),varvalue)); + return new CommandProcessorResponse(0); } else { ss.getConf().set(varname, new VariableSubstitution().substitute(ss.getConf(),varvalue) ); return new CommandProcessorResponse(0); @@ -154,8 +166,17 @@ public class SetProcessor implements CommandProcessor { ss.out.println(varname + " is undefined as a hive configuration variable"); return new CommandProcessorResponse(1); } + } else if (varname.indexOf(SetProcessor.HIVEVAR_PREFIX)==0) { + String var = varname.substring(SetProcessor.HIVEVAR_PREFIX.length()); + if (ss.getHiveVariables().get(var)!=null){ + ss.out.println(SetProcessor.HIVEVAR_PREFIX+var + "=" + ss.getHiveVariables().get(var)); + return new CommandProcessorResponse(0); + } else { + ss.out.println(varname + " is undefined as a hive variable"); + return new CommandProcessorResponse(1); + } } else { - dumpOption(ss.getConf().getAllProperties(), varname); + dumpOption(varname); return new CommandProcessorResponse(0); } } diff --git ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java index 6a6e8e8..07a4832 100644 --- ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java +++ ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java @@ -28,6 +28,7 @@ import java.util.GregorianCalendar; import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Set; import org.apache.commons.lang.StringUtils; @@ -110,6 +111,8 @@ public class SessionState { private List lastMapRedStatsList; + private Map hiveVariables; + /** * Lineage state. */ @@ -185,6 +188,17 @@ public class SessionState { return (conf.getVar(HiveConf.ConfVars.HIVEQUERYID)); } + public Map getHiveVariables() { + if (hiveVariables == null) { + hiveVariables = new HashMap(); + } + return hiveVariables; + } + + public void setHiveVariables(Map hiveVariables) { + this.hiveVariables = hiveVariables; + } + public String getSessionId() { return (conf.getVar(HiveConf.ConfVars.HIVESESSIONID)); } diff --git ql/src/test/queries/clientpositive/set_variable_sub.q ql/src/test/queries/clientpositive/set_variable_sub.q new file mode 100644 index 0000000..7bf1e60 --- /dev/null +++ ql/src/test/queries/clientpositive/set_variable_sub.q @@ -0,0 +1,15 @@ +set hivevar:key1=value1; + +EXPLAIN SELECT * FROM src where key="${key1}"; +EXPLAIN SELECT * FROM src where key="${hivevar:key1}"; + +set hivevar:a=1; +set hivevar:b=a; +set hivevar:c=${hivevar:${hivevar:b}}; +EXPLAIN SELECT * FROM src where key="${hivevar:c}"; + +set hivevar:a; +set hivevar:b; +set hivevar:c; +set hivevar:key1; + diff --git ql/src/test/results/clientpositive/set_variable_sub.q.out ql/src/test/results/clientpositive/set_variable_sub.q.out new file mode 100644 index 0000000..6067a78 --- /dev/null +++ ql/src/test/results/clientpositive/set_variable_sub.q.out @@ -0,0 +1,127 @@ +PREHOOK: query: EXPLAIN SELECT * FROM src where key="value1" +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN SELECT * FROM src where key="value1" +POSTHOOK: type: QUERY +ABSTRACT SYNTAX TREE: + (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME src))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR TOK_ALLCOLREF)) (TOK_WHERE (= (TOK_TABLE_OR_COL key) "value1")))) + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Alias -> Map Operator Tree: + src + TableScan + alias: src + Filter Operator + predicate: + expr: (key = 'value1') + type: boolean + Select Operator + expressions: + expr: key + type: string + expr: value + type: string + outputColumnNames: _col0, _col1 + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + + Stage: Stage-0 + Fetch Operator + limit: -1 + + +PREHOOK: query: EXPLAIN SELECT * FROM src where key="value1" +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN SELECT * FROM src where key="value1" +POSTHOOK: type: QUERY +ABSTRACT SYNTAX TREE: + (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME src))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR TOK_ALLCOLREF)) (TOK_WHERE (= (TOK_TABLE_OR_COL key) "value1")))) + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Alias -> Map Operator Tree: + src + TableScan + alias: src + Filter Operator + predicate: + expr: (key = 'value1') + type: boolean + Select Operator + expressions: + expr: key + type: string + expr: value + type: string + outputColumnNames: _col0, _col1 + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + + Stage: Stage-0 + Fetch Operator + limit: -1 + + +PREHOOK: query: EXPLAIN SELECT * FROM src where key="1" +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN SELECT * FROM src where key="1" +POSTHOOK: type: QUERY +ABSTRACT SYNTAX TREE: + (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME src))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR TOK_ALLCOLREF)) (TOK_WHERE (= (TOK_TABLE_OR_COL key) "1")))) + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Alias -> Map Operator Tree: + src + TableScan + alias: src + Filter Operator + predicate: + expr: (key = '1') + type: boolean + Select Operator + expressions: + expr: key + type: string + expr: value + type: string + outputColumnNames: _col0, _col1 + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + + Stage: Stage-0 + Fetch Operator + limit: -1 + + +hivevar:a=1 +hivevar:b=a +hivevar:c=1 +hivevar:key1=value1