diff --git itests/src/test/resources/testconfiguration.properties itests/src/test/resources/testconfiguration.properties index 6393671..88ef4f8 100644 --- itests/src/test/resources/testconfiguration.properties +++ itests/src/test/resources/testconfiguration.properties @@ -394,6 +394,7 @@ spark.query.files=alter_merge_orc.q \ script_env_var1.q \ script_env_var2.q \ script_pipe.q \ + scriptfile1.q \ sort.q \ spark_test.q \ temp_table.q \ diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/ScriptOperator.java ql/src/java/org/apache/hadoop/hive/ql/exec/ScriptOperator.java index 5b857e2..8921794 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/ScriptOperator.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/ScriptOperator.java @@ -50,6 +50,7 @@ import org.apache.hadoop.mapred.Reporter; import org.apache.hadoop.util.Shell; import org.apache.hadoop.util.StringUtils; +import org.apache.spark.SparkFiles; /** * ScriptOperator. @@ -188,6 +189,7 @@ public File getAbsolutePath(String filename) { if (pathenv == null || pathSep == null || fileSep == null) { return null; } + int val = -1; String classvalue = pathenv + pathSep; @@ -296,6 +298,11 @@ public void processOp(Object row, int tag) throws HiveException { if (!new File(prog).isAbsolute()) { PathFinder finder = new PathFinder("PATH"); finder.prependPathComponent(currentDir.toString()); + + // In spark local mode, we need to search added files in root directory. + if (HiveConf.getVar(hconf, HiveConf.ConfVars.HIVE_EXECUTION_ENGINE).equals("spark")) { + finder.prependPathComponent(SparkFiles.getRootDirectory()); + } File f = finder.getAbsolutePath(prog); if (f != null) { cmdArgs[0] = f.getAbsolutePath(); diff --git ql/src/test/results/clientpositive/spark/scriptfile1.q.out ql/src/test/results/clientpositive/spark/scriptfile1.q.out new file mode 100644 index 0000000..5d91e90 --- /dev/null +++ ql/src/test/results/clientpositive/spark/scriptfile1.q.out @@ -0,0 +1,50 @@ +PREHOOK: query: -- EXCLUDE_OS_WINDOWS +CREATE TABLE dest1(key INT, value STRING) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@dest1 +POSTHOOK: query: -- EXCLUDE_OS_WINDOWS +CREATE TABLE dest1(key INT, value STRING) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@dest1 +PREHOOK: query: FROM ( + FROM src + SELECT TRANSFORM(src.key, src.value) + USING 'testgrep' AS (tkey, tvalue) + CLUSTER BY tkey +) tmap +INSERT OVERWRITE TABLE dest1 SELECT tmap.tkey, tmap.tvalue +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@dest1 +[Error 30017]: Skipping stats aggregation by error org.apache.hadoop.hive.ql.metadata.HiveException: [Error 30015]: Stats aggregator of type counter cannot be connected to +POSTHOOK: query: FROM ( + FROM src + SELECT TRANSFORM(src.key, src.value) + USING 'testgrep' AS (tkey, tvalue) + CLUSTER BY tkey +) tmap +INSERT OVERWRITE TABLE dest1 SELECT tmap.tkey, tmap.tvalue +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@dest1 +POSTHOOK: Lineage: dest1.key SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: dest1.value SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: SELECT dest1.* FROM dest1 +PREHOOK: type: QUERY +PREHOOK: Input: default@dest1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT dest1.* FROM dest1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@dest1 +#### A masked pattern was here #### +10 val_10 +100 val_100 +100 val_100 +103 val_103 +103 val_103 +104 val_104 +104 val_104 +105 val_105 +310 val_310