diff --git ql/src/test/queries/clientnegative/minimr_broken_pipe.q ql/src/test/queries/clientnegative/minimr_broken_pipe.q index 8dda9cd..ec99617 100644 --- ql/src/test/queries/clientnegative/minimr_broken_pipe.q +++ ql/src/test/queries/clientnegative/minimr_broken_pipe.q @@ -1,4 +1,5 @@ set hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat; set hive.exec.script.allow.partial.consumption = false; +ADD FILE src/test/scripts/echo.py; -- Tests exception in ScriptOperator.close() by passing to the operator a small amount of data -SELECT TRANSFORM(*) USING 'true' AS a, b FROM (SELECT TRANSFORM(*) USING 'echo' AS a, b FROM src LIMIT 1) tmp; \ No newline at end of file +SELECT TRANSFORM(*) USING 'true' AS a, b FROM (SELECT TRANSFORM(*) USING 'python echo.py' AS a, b FROM src LIMIT 1) tmp; \ No newline at end of file diff --git ql/src/test/queries/clientpositive/hiveprofiler_script0.q ql/src/test/queries/clientpositive/hiveprofiler_script0.q index 3ee30d3..3c5a1c5 100644 --- ql/src/test/queries/clientpositive/hiveprofiler_script0.q +++ ql/src/test/queries/clientpositive/hiveprofiler_script0.q @@ -1,14 +1,14 @@ SET hive.task.progress=true; set hive.exec.operator.hooks=org.apache.hadoop.hive.ql.profiler.HiveProfiler; set hive.exec.post.hooks=org.apache.hadoop.hive.ql.hooks.HiveProfilerResultsHook; -ADD FILE src/test/scripts/testgrep; +ADD FILE src/test/scripts/testgrep.py; -- checking that script operator does not cause NPE -- Derby strangeness is causing the output collector for the Hive Profiler to not get output during DB read SELECT TRANSFORM(src.key, src.value) - USING 'testgrep' AS (tkey, tvalue) + USING 'python testgrep.py' AS (tkey, tvalue) FROM src diff --git ql/src/test/queries/clientpositive/scriptfile1.q ql/src/test/queries/clientpositive/scriptfile1.q index 4f65016..2593f09 100644 --- ql/src/test/queries/clientpositive/scriptfile1.q +++ ql/src/test/queries/clientpositive/scriptfile1.q @@ -1,12 +1,12 @@ set hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat; CREATE TABLE dest1(key INT, value STRING); -ADD FILE src/test/scripts/testgrep; +ADD FILE src/test/scripts/testgrep.py; FROM ( FROM src SELECT TRANSFORM(src.key, src.value) - USING 'testgrep' AS (tkey, tvalue) + USING 'python testgrep.py' AS (tkey, tvalue) CLUSTER BY tkey ) tmap INSERT OVERWRITE TABLE dest1 SELECT tmap.tkey, tmap.tvalue; diff --git ql/src/test/results/clientnegative/minimr_broken_pipe.q.out ql/src/test/results/clientnegative/minimr_broken_pipe.q.out index 5b0db2c..3e05e5f 100644 --- ql/src/test/results/clientnegative/minimr_broken_pipe.q.out +++ ql/src/test/results/clientnegative/minimr_broken_pipe.q.out @@ -1,5 +1,5 @@ PREHOOK: query: -- Tests exception in ScriptOperator.close() by passing to the operator a small amount of data -SELECT TRANSFORM(*) USING 'true' AS a, b FROM (SELECT TRANSFORM(*) USING 'echo' AS a, b FROM src LIMIT 1) tmp +SELECT TRANSFORM(*) USING 'true' AS a, b FROM (SELECT TRANSFORM(*) USING 'python echo.py' AS a, b FROM src LIMIT 1) tmp PREHOOK: type: QUERY PREHOOK: Input: default@src #### A masked pattern was here #### diff --git ql/src/test/results/clientpositive/hiveprofiler_script0.q.out ql/src/test/results/clientpositive/hiveprofiler_script0.q.out index cc03f5d..a441beb 100644 --- ql/src/test/results/clientpositive/hiveprofiler_script0.q.out +++ ql/src/test/results/clientpositive/hiveprofiler_script0.q.out @@ -2,7 +2,7 @@ PREHOOK: query: -- checking that script operator does not cause NPE -- Derby strangeness is causing the output collector for the Hive Profiler to not get output during DB read SELECT TRANSFORM(src.key, src.value) - USING 'testgrep' AS (tkey, tvalue) + USING 'python testgrep.py' AS (tkey, tvalue) FROM src PREHOOK: type: QUERY PREHOOK: Input: default@src diff --git ql/src/test/results/clientpositive/scriptfile1.q.out ql/src/test/results/clientpositive/scriptfile1.q.out index 372f85b..fa503dd 100644 --- ql/src/test/results/clientpositive/scriptfile1.q.out +++ ql/src/test/results/clientpositive/scriptfile1.q.out @@ -6,7 +6,7 @@ POSTHOOK: Output: default@dest1 PREHOOK: query: FROM ( FROM src SELECT TRANSFORM(src.key, src.value) - USING 'testgrep' AS (tkey, tvalue) + USING 'python testgrep.py' AS (tkey, tvalue) CLUSTER BY tkey ) tmap INSERT OVERWRITE TABLE dest1 SELECT tmap.tkey, tmap.tvalue @@ -16,7 +16,7 @@ PREHOOK: Output: default@dest1 POSTHOOK: query: FROM ( FROM src SELECT TRANSFORM(src.key, src.value) - USING 'testgrep' AS (tkey, tvalue) + USING 'python testgrep.py' AS (tkey, tvalue) CLUSTER BY tkey ) tmap INSERT OVERWRITE TABLE dest1 SELECT tmap.tkey, tmap.tvalue diff --git ql/src/test/scripts/echo.py ql/src/test/scripts/echo.py new file mode 100644 index 0000000..641ea57 --- /dev/null +++ ql/src/test/scripts/echo.py @@ -0,0 +1,19 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys +for s in sys.argv[1:]: + print s, diff --git ql/src/test/scripts/testgrep.py ql/src/test/scripts/testgrep.py new file mode 100644 index 0000000..0b42414 --- /dev/null +++ ql/src/test/scripts/testgrep.py @@ -0,0 +1,21 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys, re +for line in sys.stdin: + match = re.search('10.*', line) + if match: + sys.stdout.write(line);