Index: conf/hive-default.xml
===================================================================
--- conf/hive-default.xml (revision 1038444)
+++ conf/hive-default.xml (working copy)
@@ -786,4 +786,10 @@
Default property values for newly created tables
+
+ hive.variable.substitute
+ true
+ This enables substitution using syntax like ${var} ${system:var} and ${env:var}.
+
+
Index: common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
===================================================================
--- common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (revision 1038444)
+++ common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (working copy)
@@ -314,6 +314,9 @@
// Serde for FetchTask
HIVEFETCHOUTPUTSERDE("hive.fetch.output.serde", "org.apache.hadoop.hive.serde2.DelimitedJSONSerDe"),
+ // Hive Variables
+ HIVEVARIABLESUBSTITUTE("hive.variable.substitute", true),
+
SEMANTIC_ANALYZER_HOOK("hive.semantic.analyzer.hook",null),
;
@@ -621,4 +624,5 @@
public static String getColumnInternalName(int pos) {
return "_col" + pos;
}
+
}
Index: ql/src/test/results/clientpositive/set_processor_namespaces.q.out
===================================================================
--- ql/src/test/results/clientpositive/set_processor_namespaces.q.out (revision 0)
+++ ql/src/test/results/clientpositive/set_processor_namespaces.q.out (revision 0)
@@ -0,0 +1,65 @@
+zzz=5
+system:xxx=5
+system:yyy=5
+go=5
+raw=${hiveconf:zzz}
+PREHOOK: query: EXPLAIN SELECT * FROM src where key=5
+PREHOOK: type: QUERY
+POSTHOOK: query: EXPLAIN SELECT * FROM src where key=5
+POSTHOOK: type: QUERY
+ABSTRACT SYNTAX TREE:
+ (TOK_QUERY (TOK_FROM (TOK_TABREF src)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR TOK_ALLCOLREF)) (TOK_WHERE (= (TOK_TABLE_OR_COL key) 5))))
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Alias -> Map Operator Tree:
+ src
+ TableScan
+ alias: src
+ Filter Operator
+ predicate:
+ expr: (key = 5)
+ type: boolean
+ Filter Operator
+ predicate:
+ expr: (key = 5)
+ type: boolean
+ Select Operator
+ expressions:
+ expr: key
+ type: string
+ expr: value
+ type: string
+ outputColumnNames: _col0, _col1
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+
+
+PREHOOK: query: SELECT * FROM src where key=5
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: file:/tmp/edward/hive_2010-11-23_22-04-47_164_8887552189029512035/-mr-10000
+POSTHOOK: query: SELECT * FROM src where key=5
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: file:/tmp/edward/hive_2010-11-23_22-04-47_164_8887552189029512035/-mr-10000
+5 val_5
+5 val_5
+5 val_5
+c=1
+../lib/derby.jar
+/home/edward/hive-stats.txt
+/home/edward/hive-stats.txt
Index: ql/src/test/queries/clientpositive/set_processor_namespaces.q
===================================================================
--- ql/src/test/queries/clientpositive/set_processor_namespaces.q (revision 0)
+++ ql/src/test/queries/clientpositive/set_processor_namespaces.q (revision 0)
@@ -0,0 +1,32 @@
+set zzz=5;
+set zzz;
+
+set system:xxx=5;
+set system:xxx;
+
+set system:yyy=${system:xxx};
+set system:yyy;
+
+set go=${hiveconf:zzz};
+set go;
+
+set hive.variable.substitute=false;
+set raw=${hiveconf:zzz};
+set raw;
+
+set hive.variable.substitute=true;
+
+EXPLAIN SELECT * FROM src where key=${hiveconf:zzz};
+SELECT * FROM src where key=${hiveconf:zzz};
+
+set a=1;
+set b=a;
+set c=${hiveconf:${hiveconf:b}};
+set c;
+
+set jar=../lib/derby.jar;
+
+add file ${hiveconf:jar};
+list file;
+delete file ${hiveconf:jar};
+list file;
Index: ql/src/java/org/apache/hadoop/hive/ql/processors/AddResourceProcessor.java
===================================================================
--- ql/src/java/org/apache/hadoop/hive/ql/processors/AddResourceProcessor.java (revision 1038444)
+++ ql/src/java/org/apache/hadoop/hive/ql/processors/AddResourceProcessor.java (working copy)
@@ -21,6 +21,7 @@
import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hive.ql.parse.VariableSubstitution;
import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hadoop.hive.ql.session.SessionState.LogHelper;
@@ -39,6 +40,7 @@
public CommandProcessorResponse run(String command) {
SessionState ss = SessionState.get();
+ command = new VariableSubstitution().substitute(ss.getConf(),command);
String[] tokens = command.split("\\s+");
SessionState.ResourceType t;
if (tokens.length < 2
Index: ql/src/java/org/apache/hadoop/hive/ql/processors/DeleteResourceProcessor.java
===================================================================
--- ql/src/java/org/apache/hadoop/hive/ql/processors/DeleteResourceProcessor.java (revision 1038444)
+++ ql/src/java/org/apache/hadoop/hive/ql/processors/DeleteResourceProcessor.java (working copy)
@@ -21,6 +21,7 @@
import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hive.ql.parse.VariableSubstitution;
import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hadoop.hive.ql.session.SessionState.LogHelper;
@@ -38,6 +39,7 @@
public CommandProcessorResponse run(String command) {
SessionState ss = SessionState.get();
+ command = new VariableSubstitution().substitute(ss.getConf(),command);
String[] tokens = command.split("\\s+");
SessionState.ResourceType t;
Index: ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java
===================================================================
--- ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java (revision 1038444)
+++ ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java (working copy)
@@ -18,11 +18,12 @@
package org.apache.hadoop.hive.ql.processors;
+import java.util.Map;
import java.util.Properties;
-import java.util.Map;
import java.util.SortedMap;
import java.util.TreeMap;
+import org.apache.hadoop.hive.ql.parse.VariableSubstitution;
import org.apache.hadoop.hive.ql.session.SessionState;
/**
@@ -32,6 +33,9 @@
public class SetProcessor implements CommandProcessor {
private static String prefix = "set: ";
+ public static final String ENV_PREFIX = "env:";
+ public static final String SYSTEM_PREFIX = "system:";
+ public static final String HIVECONF_PREFIX = "hiveconf:";
public static boolean getBoolean(String value) {
if (value.equals("on") || value.equals("true")) {
@@ -57,6 +61,16 @@
for (Map.Entry entries : sortedMap.entrySet()) {
ss.out.println(entries.getKey() + "=" + entries.getValue());
}
+
+ for (Map.Entry entry : mapToSortedMap(System.getenv()).entrySet()) {
+ ss.out.println(ENV_PREFIX+entry.getKey() + "=" + entry.getValue());
+ }
+
+ for (Map.Entry entry :
+ propertiesToSortedMap(System.getProperties()).entrySet() ) {
+ ss.out.println(SYSTEM_PREFIX+entry.getKey() + "=" + entry.getValue());
+ }
+
}
private void dumpOption(Properties p, String s) {
@@ -72,6 +86,80 @@
public void init() {
}
+ private CommandProcessorResponse setVariable(String varname, String varvalue){
+ SessionState ss = SessionState.get();
+ if (varname.startsWith(SetProcessor.ENV_PREFIX)){
+ ss.err.println("env:* variables can not be set.");
+ return new CommandProcessorResponse(1);
+ } else if (varname.startsWith(SetProcessor.SYSTEM_PREFIX)){
+ String propName = varname.substring(SetProcessor.SYSTEM_PREFIX.length());
+ System.getProperties().setProperty(propName, new VariableSubstitution().substitute(ss.getConf(),varvalue));
+ return new CommandProcessorResponse(0);
+ } else if (varname.startsWith(SetProcessor.HIVECONF_PREFIX)){
+ String propName = varname.substring(SetProcessor.HIVECONF_PREFIX.length());
+ ss.getConf().set(propName, new VariableSubstitution().substitute(ss.getConf(),varvalue));
+ return new CommandProcessorResponse(0);
+ } else {
+ ss.getConf().set(varname, new VariableSubstitution().substitute(ss.getConf(),varvalue) );
+ return new CommandProcessorResponse(0);
+ }
+ }
+
+ private SortedMap propertiesToSortedMap(Properties p){
+ SortedMap sortedPropMap = new TreeMap();
+ for (Map.Entry