diff --git cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java index a2976b5..9afc452 100644 --- cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java +++ cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java @@ -56,8 +56,11 @@ import org.apache.hadoop.hive.ql.exec.HadoopJobExecHelper; import org.apache.hadoop.hive.ql.exec.Utilities; import org.apache.hadoop.hive.ql.exec.Utilities.StreamPrinter; import org.apache.hadoop.hive.ql.parse.ParseDriver; +import org.apache.hadoop.hive.ql.parse.Substitution; +import org.apache.hadoop.hive.ql.parse.VariableSubstitution; import org.apache.hadoop.hive.ql.processors.CommandProcessor; import org.apache.hadoop.hive.ql.processors.CommandProcessorFactory; +import org.apache.hadoop.hive.ql.processors.SetProcessor; import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hadoop.hive.ql.session.SessionState.LogHelper; import org.apache.hadoop.hive.service.HiveClient; @@ -306,10 +309,12 @@ public class CliDriver { * returning -1 * @return */ - public int processLine(String line, boolean allowInterupting) { + public int processLine(String input, boolean allowInterupting) { SignalHandler oldSignal = null; Signal interupSignal = null; + String line = preprocess(input); + if (allowInterupting) { // Remember all threads that were running at the time we started line processing. // Hook up the custom Ctrl+C handler while processing this line @@ -380,6 +385,22 @@ public class CliDriver { } } + private String preprocess(String line) { + final Map hiveVariables = SessionState.get().getHiveVariables(); + if ( hiveVariables != null ) { + return new VariableSubstitution().substitute((HiveConf) conf, new Substitution() { + @Override + public String getSubstitute(String variable) { + if (variable.startsWith(SetProcessor.HIVEVAR_PREFIX)){ + return hiveVariables.get(variable.substring(SetProcessor.HIVEVAR_PREFIX.length())); + } + return hiveVariables.get(variable); + } + },line); + } + return line; + } + public int processReader(BufferedReader r) throws IOException { String line; StringBuilder qsb = new StringBuilder(); @@ -581,6 +602,7 @@ public class CliDriver { } CliDriver cli = new CliDriver(); + cli.setHiveVariables(oproc.getHiveVariables()); // Execute -i init files (always in silent mode) cli.processInitFiles(ss); @@ -632,4 +654,8 @@ public class CliDriver { System.exit(ret); } + public void setHiveVariables(Map hiveVariables) { + SessionState.get().setHiveVariables(hiveVariables); + } + } diff --git cli/src/java/org/apache/hadoop/hive/cli/OptionsProcessor.java cli/src/java/org/apache/hadoop/hive/cli/OptionsProcessor.java index 90084ed..b6ed3b4 100644 --- cli/src/java/org/apache/hadoop/hive/cli/OptionsProcessor.java +++ cli/src/java/org/apache/hadoop/hive/cli/OptionsProcessor.java @@ -18,7 +18,9 @@ package org.apache.hadoop.hive.cli; +import java.util.HashMap; import java.util.Arrays; +import java.util.Map; import java.util.Properties; import org.apache.commons.cli.GnuParser; @@ -38,7 +40,7 @@ public class OptionsProcessor { protected static final Log l4j = LogFactory.getLog(OptionsProcessor.class.getName()); private final Options options = new Options(); private org.apache.commons.cli.CommandLine commandLine; - + Map hiveVariables = new HashMap(); @SuppressWarnings("static-access") public OptionsProcessor() { @@ -87,6 +89,15 @@ public class OptionsProcessor { .withDescription("connecting to Hive Server on port number") .create('p')); + // Substitution option -d, --define + options.addOption(OptionBuilder + .withValueSeparator() + .hasArgs(2) + .withArgName("key=value") + .withLongOpt("define") + .withDescription("Subsitution to apply to script. e.g. -d A=B or --define A=B") + .create('d')); + // [-S|--silent] options.addOption(new Option("S", "silent", false, "Silent mode in interactive shell")); @@ -105,6 +116,11 @@ public class OptionsProcessor { for (String propKey : confProps.stringPropertyNames()) { System.setProperty(propKey, confProps.getProperty(propKey)); } + + Properties hiveVars = commandLine.getOptionProperties("define"); + for (String propKey : hiveVars.stringPropertyNames()) { + hiveVariables.put(propKey, hiveVars.getProperty(propKey)); + } } catch (ParseException e) { System.err.println(e.getMessage()); printUsage(); @@ -157,4 +173,8 @@ public class OptionsProcessor { private void printUsage() { new HelpFormatter().printHelp("hive", options); } + + public Map getHiveVariables() { + return hiveVariables; + } } diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/VariableSubstitution.java ql/src/java/org/apache/hadoop/hive/ql/parse/VariableSubstitution.java index e203dda..5b8a711 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/VariableSubstitution.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/VariableSubstitution.java @@ -32,7 +32,34 @@ public class VariableSubstitution { protected static Pattern varPat = Pattern.compile("\\$\\{[^\\}\\$\u0020]+\\}"); protected static int MAX_SUBST = 40; - public String substitute (HiveConf conf, String expr) { + public String substitute (final HiveConf conf, String expr) { + return substitute(conf, new Substitution() { + @Override + public String getSubstitute(String var) { + String val = null; + try { + if (var.startsWith(SetProcessor.SYSTEM_PREFIX)) { + val = System.getProperty(var.substring(SetProcessor.SYSTEM_PREFIX.length())); + } + } catch(SecurityException se) { + l4j.warn("Unexpected SecurityException in Configuration", se); + } + if (val ==null){ + if (var.startsWith(SetProcessor.ENV_PREFIX)){ + val = System.getenv(var.substring(SetProcessor.ENV_PREFIX.length())); + } + } + if (val == null) { + if (var.startsWith(SetProcessor.HIVECONF_PREFIX)){ + val = conf.get(var.substring(SetProcessor.HIVECONF_PREFIX.length())); + } + } + return val; + } + }, expr); + } + + public String substitute (HiveConf conf, Substitution substitutes, String expr) { if (conf.getBoolVar(ConfVars.HIVEVARIABLESUBSTITUTE)){ l4j.debug("Substitution is on: "+expr); @@ -52,26 +79,14 @@ public class VariableSubstitution { String var = match.group(); var = var.substring(2, var.length()-1); // remove ${ .. } String val = null; - try { - if (var.startsWith(SetProcessor.SYSTEM_PREFIX)) { - val = System.getProperty(var.substring(SetProcessor.SYSTEM_PREFIX.length())); - } - } catch(SecurityException se) { - l4j.warn("Unexpected SecurityException in Configuration", se); - } - if (val ==null){ - if (var.startsWith(SetProcessor.ENV_PREFIX)){ - val = System.getenv(var.substring(SetProcessor.ENV_PREFIX.length())); - } - } - if (val == null) { - if (var.startsWith(SetProcessor.HIVECONF_PREFIX)){ - val = conf.get(var.substring(SetProcessor.HIVECONF_PREFIX.length())); - } + + if (substitutes != null) { + // Try variable substitution + val = substitutes.getSubstitute(var); } if (val == null) { l4j.debug("Interpolation result: "+eval); - return eval; // return literal ${var}: var is unbound + return eval; // return literal, no substitution found } // substitute eval = eval.substring(0, match.start())+val+eval.substring(match.end()); diff --git ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java index 97fa1ab..e8e57be 100644 --- ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java +++ ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java @@ -36,6 +36,7 @@ public class SetProcessor implements CommandProcessor { public static final String ENV_PREFIX = "env:"; public static final String SYSTEM_PREFIX = "system:"; public static final String HIVECONF_PREFIX = "hiveconf:"; + public static final String HIVEVAR_PREFIX = "define:"; public static boolean getBoolean(String value) { if (value.equals("on") || value.equals("true")) { @@ -58,6 +59,12 @@ public class SetProcessor implements CommandProcessor { String oneValue = p.getProperty(oneProp); sortedMap.put(oneProp, oneValue); } + + // Inserting hive variables + for (String s : ss.getHiveVariables().keySet()) { + sortedMap.put(SetProcessor.HIVEVAR_PREFIX + s, ss.getHiveVariables().get(s)); + } + for (Map.Entry entries : sortedMap.entrySet()) { ss.out.println(entries.getKey() + "=" + entries.getValue()); } @@ -73,11 +80,13 @@ public class SetProcessor implements CommandProcessor { } - private void dumpOption(Properties p, String s) { + private void dumpOption(String s) { SessionState ss = SessionState.get(); - if (p.getProperty(s) != null) { - ss.out.println(s + "=" + p.getProperty(s)); + if (ss.getConf().get(s) != null) { + ss.out.println(s + "=" + ss.getConf().get(s)); + } else if (ss.getHiveVariables().containsKey(s)) { + ss.out.println(s + "=" + ss.getHiveVariables().get(s)); } else { ss.out.println(s + " is undefined"); } @@ -99,6 +108,10 @@ public class SetProcessor implements CommandProcessor { String propName = varname.substring(SetProcessor.HIVECONF_PREFIX.length()); ss.getConf().set(propName, new VariableSubstitution().substitute(ss.getConf(),varvalue)); return new CommandProcessorResponse(0); + } else if (varname.startsWith(SetProcessor.HIVEVAR_PREFIX)) { + String propName = varname.substring(SetProcessor.HIVEVAR_PREFIX.length()); + ss.getHiveVariables().put(propName, new VariableSubstitution().substitute(ss.getConf(),varvalue)); + return new CommandProcessorResponse(0); } else { ss.getConf().set(varname, new VariableSubstitution().substitute(ss.getConf(),varvalue) ); return new CommandProcessorResponse(0); @@ -154,8 +167,17 @@ public class SetProcessor implements CommandProcessor { ss.out.println(varname + " is undefined as a hive configuration variable"); return new CommandProcessorResponse(1); } + } else if (varname.indexOf(SetProcessor.HIVEVAR_PREFIX)==0) { + String var = varname.substring(SetProcessor.HIVEVAR_PREFIX.length()); + if (ss.getHiveVariables().get(var)!=null){ + ss.out.println(SetProcessor.HIVEVAR_PREFIX+var + "=" + ss.getHiveVariables().get(var)); + return new CommandProcessorResponse(0); + } else { + ss.out.println(varname + " is undefined as a hive variable"); + return new CommandProcessorResponse(1); + } } else { - dumpOption(ss.getConf().getAllProperties(), varname); + dumpOption(varname); return new CommandProcessorResponse(0); } } diff --git ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java index 6a6e8e8..07a4832 100644 --- ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java +++ ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java @@ -28,6 +28,7 @@ import java.util.GregorianCalendar; import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Set; import org.apache.commons.lang.StringUtils; @@ -110,6 +111,8 @@ public class SessionState { private List lastMapRedStatsList; + private Map hiveVariables; + /** * Lineage state. */ @@ -185,6 +188,17 @@ public class SessionState { return (conf.getVar(HiveConf.ConfVars.HIVEQUERYID)); } + public Map getHiveVariables() { + if (hiveVariables == null) { + hiveVariables = new HashMap(); + } + return hiveVariables; + } + + public void setHiveVariables(Map hiveVariables) { + this.hiveVariables = hiveVariables; + } + public String getSessionId() { return (conf.getVar(HiveConf.ConfVars.HIVESESSIONID)); }