Index: ql/src/test/results/clientpositive/scriptfile2.q.out =================================================================== --- ql/src/test/results/clientpositive/scriptfile2.q.out (revision 0) +++ ql/src/test/results/clientpositive/scriptfile2.q.out (revision 0) @@ -0,0 +1,46 @@ +PREHOOK: query: CREATE TABLE dest1(key INT, value STRING) +PREHOOK: type: CREATETABLE +POSTHOOK: query: CREATE TABLE dest1(key INT, value STRING) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@dest1 +PREHOOK: query: FROM ( + FROM src + SELECT TRANSFORM(src.key, src.value) + USING 'testgrep' AS (tkey, tvalue) + CLUSTER BY tkey +) tmap +INSERT OVERWRITE TABLE dest1 SELECT tmap.tkey, tmap.tvalue +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@dest1 +POSTHOOK: query: FROM ( + FROM src + SELECT TRANSFORM(src.key, src.value) + USING 'testgrep' AS (tkey, tvalue) + CLUSTER BY tkey +) tmap +INSERT OVERWRITE TABLE dest1 SELECT tmap.tkey, tmap.tvalue +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@dest1 +POSTHOOK: Lineage: dest1.key SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: dest1.value SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: SELECT dest1.* FROM dest1 +PREHOOK: type: QUERY +PREHOOK: Input: default@dest1 +PREHOOK: Output: file:/tmp/jssarma/hive_2010-10-04_14-59-27_464_1003308786334092030/-mr-10000 +POSTHOOK: query: SELECT dest1.* FROM dest1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@dest1 +POSTHOOK: Output: file:/tmp/jssarma/hive_2010-10-04_14-59-27_464_1003308786334092030/-mr-10000 +POSTHOOK: Lineage: dest1.key SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: dest1.value SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ] +10 val_10 +100 val_100 +100 val_100 +103 val_103 +103 val_103 +104 val_104 +104 val_104 +105 val_105 +310 val_310 Index: ql/src/test/queries/clientpositive/scriptfile2.q =================================================================== --- ql/src/test/queries/clientpositive/scriptfile2.q (revision 0) +++ ql/src/test/queries/clientpositive/scriptfile2.q (revision 0) @@ -0,0 +1,15 @@ +set mapred.job.tracker=local; + +CREATE TABLE dest1(key INT, value STRING); + +ADD FILE src/test/scripts/testgrep; + +FROM ( + FROM src + SELECT TRANSFORM(src.key, src.value) + USING 'testgrep' AS (tkey, tvalue) + CLUSTER BY tkey +) tmap +INSERT OVERWRITE TABLE dest1 SELECT tmap.tkey, tmap.tvalue; + +SELECT dest1.* FROM dest1; Index: ql/src/test/queries/clientpositive/scriptfile1.q =================================================================== --- ql/src/test/queries/clientpositive/scriptfile1.q (revision 1004355) +++ ql/src/test/queries/clientpositive/scriptfile1.q (working copy) @@ -1,4 +1,4 @@ -CREATE TABLE scriptfile1_dest1(key INT, value STRING); +CREATE TABLE dest1(key INT, value STRING); ADD FILE src/test/scripts/testgrep; @@ -8,6 +8,6 @@ USING 'testgrep' AS (tkey, tvalue) CLUSTER BY tkey ) tmap -INSERT OVERWRITE TABLE scriptfile1_dest1 SELECT tmap.tkey, tmap.tvalue; +INSERT OVERWRITE TABLE dest1 SELECT tmap.tkey, tmap.tvalue; -SELECT scriptfile1_dest1.* FROM scriptfile1_dest1; +SELECT dest1.* FROM dest1; Index: ql/src/java/org/apache/hadoop/hive/ql/exec/MapRedTask.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/exec/MapRedTask.java (revision 1004355) +++ ql/src/java/org/apache/hadoop/hive/ql/exec/MapRedTask.java (working copy) @@ -19,6 +19,7 @@ package org.apache.hadoop.hive.ql.exec; import java.io.IOException; +import java.io.File; import java.io.OutputStream; import java.io.Serializable; import java.util.HashMap; @@ -28,6 +29,7 @@ import org.apache.commons.lang.StringUtils; import org.apache.hadoop.fs.ContentSummary; import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; @@ -164,9 +166,23 @@ String cmdLine = hadoopExec + " jar " + jarCmd + " -plan " + planPath.toString() + " " + isSilent + " " + hiveConfArgs; + String workDir = (new File(".")).getCanonicalPath(); String files = getResourceFiles(conf, SessionState.ResourceType.FILE); if (!files.isEmpty()) { cmdLine = cmdLine + " -files " + files; + + workDir = (new Path(ctx.getLocalTmpFileURI())).toUri().getPath(); + + if (! (new File(workDir)).mkdir()) + throw new IOException ("Cannot create tmp working dir: " + workDir); + + for (String f: StringUtils.split(files, ',')) { + Path p = new Path(f); + String target = p.toUri().getPath(); + String link = workDir + Path.SEPARATOR + p.getName(); + if (FileUtil.symLink(target, link) != 0) + throw new IOException ("Cannot link to added file: " + f + " from: " + link); + } } LOG.info("Executing: " + cmdLine); @@ -207,7 +223,7 @@ env[pos++] = name + "=" + value; } // Run ExecDriver in another JVM - executor = Runtime.getRuntime().exec(cmdLine, env); + executor = Runtime.getRuntime().exec(cmdLine, env, new File(workDir)); StreamPrinter outPrinter = new StreamPrinter(executor.getInputStream(), null, System.out); Index: ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java (revision 1004355) +++ ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java (working copy) @@ -1373,37 +1373,6 @@ return HiveConf.getColumnInternalName(pos); } - /** - * If the user script command needs any modifications - do it here. - */ - private String getFixedCmd(String cmd) { - SessionState ss = SessionState.get(); - if (ss == null) { - return cmd; - } - - // for local mode - replace any references to packaged files by name with - // the reference to the original file path - if (ss.getConf().get("mapred.job.tracker", "local").equals("local")) { - Set files = ss - .list_resource(SessionState.ResourceType.FILE, null); - if ((files != null) && !files.isEmpty()) { - String prog = getScriptProgName(cmd); - String args = getScriptArgs(cmd); - - for (String oneFile : files) { - Path p = new Path(oneFile); - if (p.getName().equals(prog)) { - cmd = oneFile + args; - break; - } - } - } - } - - return cmd; - } - private String getScriptProgName(String cmd) { int end = cmd.indexOf(" "); return (end == -1) ? cmd : cmd.substring(0, end); @@ -1667,7 +1636,7 @@ Operator output = putOpInsertMap(OperatorFactory.getAndMakeChild( new ScriptDesc( - getFixedCmd(fetchFilesNotInLocalFilesystem(stripQuotes(trfm.getChild(execPos).getText()))), + fetchFilesNotInLocalFilesystem(stripQuotes(trfm.getChild(execPos).getText())), inInfo, inRecordWriter, outInfo, outRecordReader, errRecordReader, errInfo), new RowSchema(out_rwsch.getColumnInfos()), input), out_rwsch);