Index: conf/hive-default.xml =================================================================== --- conf/hive-default.xml (revision 955109) +++ conf/hive-default.xml (working copy) @@ -583,4 +583,10 @@ numbers, this conf var needs to be set manually. + + hive.variable.interpolate + true + Interpolation does substituation using ${var} ${system:var} and ${env:var}. + + Index: common/src/java/org/apache/hadoop/hive/conf/HiveConf.java =================================================================== --- common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (revision 955109) +++ common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (working copy) @@ -24,6 +24,8 @@ import java.util.Iterator; import java.util.Map; import java.util.Properties; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import javax.security.auth.login.LoginException; @@ -31,6 +33,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hadoop.hive.shims.ShimLoader; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.security.UserGroupInformation; @@ -44,6 +47,8 @@ protected Properties origProp; protected String auxJars; private static final Log l4j = LogFactory.getLog(HiveConf.class); + protected Pattern varPat = Pattern.compile("\\$\\{[^\\}\\$\u0020]+\\}"); + protected static int MAX_SUBST = 40; /** * Metastore related options that the db is initialized against. @@ -250,6 +255,9 @@ // For har files HIVEARCHIVEENABLED("hive.archive.enabled", false), HIVEHARPARENTDIRSETTABLE("hive.archive.har.parentdir.settable", false), + + //variable interpolation + HIVEVARIABLEINTERPOLATE("hive.variable.interpolate", true), ; public final String varname; @@ -547,4 +555,51 @@ public static String getColumnInternalName(int pos) { return "_col" + pos; } + + public String interpolate (String expr) { + SessionState ss = SessionState.get(); + if ( getBoolVar(ConfVars.HIVEVARIABLEINTERPOLATE)==true){ + l4j.info("Interpolation is on: "+expr); + } else { + return expr; + } + if (expr == null) { + return null; + } + Matcher match = varPat.matcher(""); + String eval = expr; + for(int s=0; s Map Operator Tree: + src + TableScan + alias: src + Filter Operator + predicate: + expr: (key = 5) + type: boolean + Filter Operator + predicate: + expr: (key = 5) + type: boolean + Select Operator + expressions: + expr: key + type: string + expr: value + type: string + outputColumnNames: _col0, _col1 + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + + Stage: Stage-0 + Fetch Operator + limit: -1 + + +PREHOOK: query: SELECT * FROM src where key=5 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: file:/mnt/data/hive/hive/build/ql/scratchdir/hive_2010-06-23_07-50-37_779_1688047116050540300/10000 +POSTHOOK: query: SELECT * FROM src where key=5 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: file:/mnt/data/hive/hive/build/ql/scratchdir/hive_2010-06-23_07-50-37_779_1688047116050540300/10000 +5 val_5 +5 val_5 +5 val_5 Index: ql/src/test/queries/clientpositive/set_processor_namespaces.q =================================================================== --- ql/src/test/queries/clientpositive/set_processor_namespaces.q (revision 0) +++ ql/src/test/queries/clientpositive/set_processor_namespaces.q (revision 0) @@ -0,0 +1,18 @@ +set zzz=5; +set zzz; + +set system:xxx=5; +set system:xxx; + +set go=${zzz}; +set go; + +set hive.variable.interpolate=false; +set raw=${zzz}; +set raw; + +set hive.variable.interpolate=true; + +EXPLAIN SELECT * FROM src where key=${zzz}; +SELECT * FROM src where key=${zzz}; + Index: ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java (revision 955109) +++ ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java (working copy) @@ -18,8 +18,8 @@ package org.apache.hadoop.hive.ql.processors; +import java.util.Map; import java.util.Properties; -import java.util.Map; import java.util.SortedMap; import java.util.TreeMap; @@ -31,6 +31,7 @@ */ public class SetProcessor implements CommandProcessor { + private static String prefix = "set: "; public static boolean getBoolean(String value) { @@ -57,6 +58,15 @@ for (Map.Entry entries : sortedMap.entrySet()) { ss.out.println(entries.getKey() + "=" + entries.getValue()); } + + for (Map.Entry entry : mapToSortedMap(System.getenv()).entrySet()) { + ss.out.println("env:"+entry.getKey() + "=" + entry.getValue()); + } + + for (Map.Entry entry : + propertiesToSortedMap(System.getProperties()).entrySet() ) { + ss.out.println("system:"+entry.getKey() + "=" + entry.getValue()); + } } private void dumpOption(Properties p, String s) { @@ -72,6 +82,67 @@ public void init() { } + private CommandProcessorResponse setVariable(String varname, String varvalue){ + SessionState ss = SessionState.get(); + if (varname.startsWith("env:")){ + ss.err.println("env:* variables can not be set."); + return new CommandProcessorResponse(1); + } else if (varname.startsWith("system:")){ + String propName = varname.substring(7); + System.getProperties().setProperty(propName, varvalue); + return new CommandProcessorResponse(0); + } else { + ss.getConf().set(varname, ss.getConf().interpolate(varvalue) ); + return new CommandProcessorResponse(0); + } + } + + private SortedMap propertiesToSortedMap(Properties p){ + SortedMap sortedPropMap = new TreeMap(); + for (Map.Entry entry :System.getProperties().entrySet() ){ + sortedPropMap.put( (String) entry.getKey(), (String) entry.getValue()); + } + return sortedPropMap; + } + + private SortedMap mapToSortedMap(Map data){ + SortedMap sortedEnvMap = new TreeMap(); + sortedEnvMap.putAll( data ); + return sortedEnvMap; + } + + + private CommandProcessorResponse getVariable(String varname){ + SessionState ss = SessionState.get(); + if (varname.equals("silent")){ + ss.out.println("silent" + "=" + ss.getIsSilent()); + return new CommandProcessorResponse(0); + } + if (varname.startsWith("system:")){ + String propName = varname.substring(7); + String result = System.getProperty(propName); + if (result != null){ + ss.out.println("system:"+propName + "=" + result); + return new CommandProcessorResponse(0); + } else { + ss.out.println( propName + " is undefined as a system property"); + return new CommandProcessorResponse(1); + } + } else if ( varname.indexOf("env:")==0){ + String var = varname.substring(4); + if ( System.getenv(var)!=null){ + ss.out.println("env:"+var + "=" + System.getenv(var)); + return new CommandProcessorResponse(0); + } else { + ss.out.println(varname + " is undefined as an environmental variable"); + return new CommandProcessorResponse(1); + } + } else { + dumpOption(ss.getConf().getAllProperties(), varname); + return new CommandProcessorResponse(0); + } + } + public CommandProcessorResponse run(String command) { SessionState ss = SessionState.get(); @@ -87,31 +158,24 @@ } String[] part = new String[2]; - int eqIndex = nwcmd.indexOf('='); - if (eqIndex == -1) { - // no equality sign - print the property out - dumpOption(ss.getConf().getAllProperties(), nwcmd); - return new CommandProcessorResponse(0); - } else if (eqIndex == nwcmd.length() - 1) { - part[0] = nwcmd.substring(0, nwcmd.length() - 1); - part[1] = ""; - } else { - part[0] = nwcmd.substring(0, eqIndex).trim(); - part[1] = nwcmd.substring(eqIndex + 1).trim(); - } - try { + if (nwcmd.contains("=")){ + if (eqIndex == nwcmd.length() - 1) { //x= + part[0] = nwcmd.substring(0, nwcmd.length() - 1); + part[1] = ""; + } else { //x=y + part[0] = nwcmd.substring(0, eqIndex).trim(); + part[1] = nwcmd.substring(eqIndex + 1).trim(); + } if (part[0].equals("silent")) { - boolean val = getBoolean(part[1]); - ss.setIsSilent(val); - } else { - ss.getConf().set(part[0], part[1]); + ss.setIsSilent(getBoolean(part[1])); + return new CommandProcessorResponse(0); } - } catch (IllegalArgumentException err) { - ss.err.println(err.getMessage()); - return new CommandProcessorResponse(1); + return setVariable(part[0],part[1]); + } else { + return getVariable(nwcmd); } - return new CommandProcessorResponse(0); + } } Index: ql/src/java/org/apache/hadoop/hive/ql/Driver.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/Driver.java (revision 955109) +++ ql/src/java/org/apache/hadoop/hive/ql/Driver.java (working copy) @@ -293,6 +293,7 @@ TaskFactory.resetId(); try { + command = conf.interpolate(command); ctx = new Context(conf); ParseDriver pd = new ParseDriver();