diff --git a/data/conf/hive-log4j.properties b/data/conf/hive-log4j.properties index f90b833..922efb8 100644 --- a/data/conf/hive-log4j.properties +++ b/data/conf/hive-log4j.properties @@ -42,7 +42,7 @@ log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout # Pattern format: Date LogLevel LoggerName LogMessage #log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n # Debugging Pattern format -log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n +log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p [%t]: %c{2} (%F:%M(%L)) - %m%n # diff --git a/data/conf/spark/hive-site.xml b/data/conf/spark/hive-site.xml index f3d7b40..59c4cd9 100644 --- a/data/conf/spark/hive-site.xml +++ b/data/conf/spark/hive-site.xml @@ -197,7 +197,12 @@ spark.master - local[4] + local-cluster[2,2,1024] + + + + spark.akka.logLifecycleEvents + true diff --git a/itests/qtest-spark/.gitignore b/itests/qtest-spark/.gitignore new file mode 100644 index 0000000..c2ed135 --- /dev/null +++ b/itests/qtest-spark/.gitignore @@ -0,0 +1 @@ +thirdparty/ diff --git a/itests/qtest-spark/pom.xml b/itests/qtest-spark/pom.xml index a533e8d..73febda 100644 --- a/itests/qtest-spark/pom.xml +++ b/itests/qtest-spark/pom.xml @@ -339,6 +339,42 @@ maven-antrun-plugin + download-spark + generate-sources + + run + + + + + env + set -e + set -x + /bin/pwd + BASE_DIR=./target + DOWNLOAD_DIR=./thirdparty + download() { + url=$1; + finalName=$2 + tarName=$(basename $url) + rm -rf $BASE_DIR/$finalName + if [[ ! -f $DOWNLOAD_DIR/$tarName ]] + then + curl -Sso $DOWNLOAD_DIR/$tarName $url + fi + tar -zxf $DOWNLOAD_DIR/$tarName -C $BASE_DIR + mv $BASE_DIR/${finalName}* $BASE_DIR/$finalName + } + mkdir -p $DOWNLOAD_DIR + download "http://ec2-50-18-79-139.us-west-1.compute.amazonaws.com/data/spark-1.2.1-SNAPSHOT-bin-hadoop2-without-hive.tgz" "spark" + + + + + + + + generate-tests-sources generate-test-sources diff --git a/pom.xml b/pom.xml index cce2e5e..900dddb 100644 --- a/pom.xml +++ b/pom.xml @@ -171,6 +171,7 @@ 2.4.0 2.6.0 3.0.0 + ${basedir}/${hive.path.to.root}/itests/qtest-spark/target/spark @@ -836,6 +837,7 @@ true ${test.tmp.dir} + ${spark.home} ${test.tmp.dir} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/SparkJobMonitor.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/SparkJobMonitor.java index 72108cd..c075c35 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/SparkJobMonitor.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/SparkJobMonitor.java @@ -67,6 +67,9 @@ public int startMonitor() { while (true) { try { JobExecutionStatus state = sparkJobStatus.getState(); + if (LOG.isDebugEnabled()) { + console.printInfo("state = " + state); + } if (state != null && state != JobExecutionStatus.UNKNOWN && (state != lastState || state == JobExecutionStatus.RUNNING)) { lastState = state; diff --git a/ql/src/test/queries/clientpositive/auto_join_without_localtask.q b/ql/src/test/queries/clientpositive/auto_join_without_localtask.q index f23e227..91aa963 100644 --- a/ql/src/test/queries/clientpositive/auto_join_without_localtask.q +++ b/ql/src/test/queries/clientpositive/auto_join_without_localtask.q @@ -5,14 +5,14 @@ set hive.auto.convert.join.use.nonstaged=true; set hive.auto.convert.join.noconditionaltask.size=100; explain -select a.* from src a join src b on a.key=b.key limit 40; +select a.* from src a join src b on a.key=b.key order by key, value limit 40; -select a.* from src a join src b on a.key=b.key limit 40; +select a.* from src a join src b on a.key=b.key order by key, value limit 40; explain -select a.* from src a join src b on a.key=b.key join src c on a.value=c.value limit 40; +select a.* from src a join src b on a.key=b.key join src c on a.value=c.value order by a.key, a.value limit 40; -select a.* from src a join src b on a.key=b.key join src c on a.value=c.value limit 40; +select a.* from src a join src b on a.key=b.key join src c on a.value=c.value order by a.key, a.value limit 40; set hive.auto.convert.join.noconditionaltask.size=100; diff --git a/ql/src/test/queries/clientpositive/count.q b/ql/src/test/queries/clientpositive/count.q index 74ae9e4..18721e5 100644 --- a/ql/src/test/queries/clientpositive/count.q +++ b/ql/src/test/queries/clientpositive/count.q @@ -1,3 +1,4 @@ +-- SORT_QUERY_RESULTS create table abcd (a int, b int, c int, d int); LOAD DATA LOCAL INPATH '../../data/files/in4.txt' INTO TABLE abcd; diff --git a/ql/src/test/queries/clientpositive/limit_pushdown.q b/ql/src/test/queries/clientpositive/limit_pushdown.q index ba52af4..4cbe501 100644 --- a/ql/src/test/queries/clientpositive/limit_pushdown.q +++ b/ql/src/test/queries/clientpositive/limit_pushdown.q @@ -14,8 +14,8 @@ select key,value from src order by key desc limit 20; select key,value from src order by key desc limit 20; explain -select value, sum(key + 1) as sum from src group by value limit 20; -select value, sum(key + 1) as sum from src group by value limit 20; +select value, sum(key + 1) as sum from src group by value order by value limit 20; +select value, sum(key + 1) as sum from src group by value order by value limit 20; -- deduped RS explain @@ -28,13 +28,13 @@ select distinct(cdouble) from alltypesorc limit 20; select distinct(cdouble) from alltypesorc limit 20; explain -select ctinyint, count(distinct(cdouble)) from alltypesorc group by ctinyint limit 20; -select ctinyint, count(distinct(cdouble)) from alltypesorc group by ctinyint limit 20; +select ctinyint, count(distinct(cdouble)) from alltypesorc group by ctinyint order by ctinyint limit 20; +select ctinyint, count(distinct(cdouble)) from alltypesorc group by ctinyint order by ctinyint limit 20; -- multi distinct explain -select ctinyint, count(distinct(cstring1)), count(distinct(cstring2)) from alltypesorc group by ctinyint limit 20; -select ctinyint, count(distinct(cstring1)), count(distinct(cstring2)) from alltypesorc group by ctinyint limit 20; +select ctinyint, count(distinct(cstring1)), count(distinct(cstring2)) from alltypesorc group by ctinyint order by ctinyint limit 20; +select ctinyint, count(distinct(cstring1)), count(distinct(cstring2)) from alltypesorc group by ctinyint order by ctinyint limit 20; -- limit zero explain @@ -57,8 +57,8 @@ on subq.key=subq2.key limit 4; set hive.map.aggr=false; -- map aggregation disabled explain -select value, sum(key) as sum from src group by value limit 20; -select value, sum(key) as sum from src group by value limit 20; +select value, sum(key) as sum from src group by value order by value limit 20; +select value, sum(key) as sum from src group by value order by value limit 20; set hive.limit.pushdown.memory.usage=0.00002f; diff --git a/ql/src/test/queries/clientpositive/mapreduce2.q b/ql/src/test/queries/clientpositive/mapreduce2.q index ef6f8e5..7539d3f 100644 --- a/ql/src/test/queries/clientpositive/mapreduce2.q +++ b/ql/src/test/queries/clientpositive/mapreduce2.q @@ -14,4 +14,4 @@ MAP src.key, CAST(src.key / 10 AS INT), CAST(src.key % 10 AS INT), src.value USING 'cat' AS (tkey, ten, one, tvalue) DISTRIBUTE BY tvalue, tkey; -SELECT * FROM (SELECT dest1.* FROM dest1 DISTRIBUTE BY key SORT BY key, ten, one, value) T; +SELECT * FROM (SELECT dest1.* FROM dest1 DISTRIBUTE BY key SORT BY key, ten, one, value) T ORDER BY key; diff --git a/ql/src/test/queries/clientpositive/multi_insert_gby3.q b/ql/src/test/queries/clientpositive/multi_insert_gby3.q index 040adca..1221af4 100644 --- a/ql/src/test/queries/clientpositive/multi_insert_gby3.q +++ b/ql/src/test/queries/clientpositive/multi_insert_gby3.q @@ -1,3 +1,4 @@ +-- SORT_QUERY_RESULTS create table e1 (key string, keyD double); create table e2 (key string, keyD double, value string); create table e3 (key string, keyD double); diff --git a/ql/src/test/queries/clientpositive/multi_join_union.q b/ql/src/test/queries/clientpositive/multi_join_union.q index 39650fd..c555b38 100644 --- a/ql/src/test/queries/clientpositive/multi_join_union.q +++ b/ql/src/test/queries/clientpositive/multi_join_union.q @@ -1,6 +1,7 @@ - set hive.auto.convert.join=true; +-- SORT_QUERY_RESULTS + CREATE TABLE src11 as SELECT * FROM src; CREATE TABLE src12 as SELECT * FROM src; CREATE TABLE src13 as SELECT * FROM src; @@ -10,4 +11,4 @@ CREATE TABLE src14 as SELECT * FROM src; EXPLAIN SELECT * FROM src11 a JOIN src12 b ON (a.key = b.key) JOIN -(SELECT * FROM (SELECT * FROM src13 UNION ALL SELECT * FROM src14)a )c ON c.value = b.value; \ No newline at end of file +(SELECT * FROM (SELECT * FROM src13 UNION ALL SELECT * FROM src14)a )c ON c.value = b.value; diff --git a/ql/src/test/queries/clientpositive/ppd_outer_join3.q b/ql/src/test/queries/clientpositive/ppd_outer_join3.q index d756b90..03169a4 100644 --- a/ql/src/test/queries/clientpositive/ppd_outer_join3.q +++ b/ql/src/test/queries/clientpositive/ppd_outer_join3.q @@ -1,6 +1,8 @@ set hive.optimize.ppd=true; set hive.ppd.remove.duplicatefilters=false; +-- SORT_QUERY_RESULTS + EXPLAIN FROM src a diff --git a/ql/src/test/queries/clientpositive/ptf_decimal.q b/ql/src/test/queries/clientpositive/ptf_decimal.q index d48dd36..94885e3 100644 --- a/ql/src/test/queries/clientpositive/ptf_decimal.q +++ b/ql/src/test/queries/clientpositive/ptf_decimal.q @@ -1,3 +1,4 @@ +-- SORT_QUERY_RESULTS -- 1. aggregate functions with decimal type diff --git a/ql/src/test/queries/clientpositive/ptf_general_queries.q b/ql/src/test/queries/clientpositive/ptf_general_queries.q index fcb2e41..9942063 100644 --- a/ql/src/test/queries/clientpositive/ptf_general_queries.q +++ b/ql/src/test/queries/clientpositive/ptf_general_queries.q @@ -1,3 +1,5 @@ +-- SORT_QUERY_RESULTS + -- 1. testNoPTFNoWindowing select p_mfgr, p_name, p_size from part diff --git a/ql/src/test/results/clientpositive/spark/auto_join20.q.out b/ql/src/test/results/clientpositive/spark/auto_join20.q.out index e089234..31ee1bc 100644 --- a/ql/src/test/results/clientpositive/spark/auto_join20.q.out +++ b/ql/src/test/results/clientpositive/spark/auto_join20.q.out @@ -249,7 +249,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (SORT, 1) + Reducer 2 <- Map 1 (SORT, 3) Reducer 3 <- Reducer 2 (GROUP, 1) #### A masked pattern was here #### Vertices: diff --git a/ql/src/test/results/clientpositive/spark/auto_join29.q.out b/ql/src/test/results/clientpositive/spark/auto_join29.q.out index 6702c1c..e618ce0 100644 --- a/ql/src/test/results/clientpositive/spark/auto_join29.q.out +++ b/ql/src/test/results/clientpositive/spark/auto_join29.q.out @@ -694,7 +694,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 4 <- Map 3 (SORT, 1) + Reducer 4 <- Map 3 (SORT, 3) #### A masked pattern was here #### Vertices: Map 3 @@ -1327,7 +1327,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (SORT, 1) + Reducer 2 <- Map 1 (SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -1969,7 +1969,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 3 <- Map 2 (SORT, 1) + Reducer 3 <- Map 2 (SORT, 3) #### A masked pattern was here #### Vertices: Map 2 @@ -2606,7 +2606,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 4 <- Map 3 (SORT, 1) + Reducer 4 <- Map 3 (SORT, 3) #### A masked pattern was here #### Vertices: Map 3 @@ -2738,7 +2738,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 3 <- Map 2 (SORT, 1) + Reducer 3 <- Map 2 (SORT, 3) #### A masked pattern was here #### Vertices: Map 2 @@ -3371,7 +3371,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 4 <- Map 3 (SORT, 1) + Reducer 4 <- Map 3 (SORT, 3) #### A masked pattern was here #### Vertices: Map 3 @@ -3504,7 +3504,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (SORT, 1) + Reducer 2 <- Map 1 (SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -3654,7 +3654,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 4 <- Map 3 (SORT, 1) + Reducer 4 <- Map 3 (SORT, 3) #### A masked pattern was here #### Vertices: Map 3 diff --git a/ql/src/test/results/clientpositive/spark/auto_join30.q.out b/ql/src/test/results/clientpositive/spark/auto_join30.q.out index f6dd445..b401af5 100644 --- a/ql/src/test/results/clientpositive/spark/auto_join30.q.out +++ b/ql/src/test/results/clientpositive/spark/auto_join30.q.out @@ -185,7 +185,7 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 5 <- Map 4 (SORT, 1) + Reducer 5 <- Map 4 (SORT, 3) #### A masked pattern was here #### Vertices: Map 4 @@ -221,7 +221,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (SORT, 1) + Reducer 2 <- Map 1 (SORT, 3) Reducer 3 <- Reducer 2 (GROUP, 1) #### A masked pattern was here #### Vertices: @@ -341,7 +341,7 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 2 <- Map 1 (SORT, 1) + Reducer 2 <- Map 1 (SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -376,7 +376,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 4 <- Map 3 (SORT, 1) + Reducer 4 <- Map 3 (SORT, 3) Reducer 5 <- Reducer 4 (GROUP, 1) #### A masked pattern was here #### Vertices: @@ -503,8 +503,8 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 2 <- Map 1 (SORT, 1) - Reducer 7 <- Map 6 (SORT, 1) + Reducer 2 <- Map 1 (SORT, 3) + Reducer 7 <- Map 6 (SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -579,7 +579,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 4 <- Map 3 (SORT, 1) + Reducer 4 <- Map 3 (SORT, 3) Reducer 5 <- Reducer 4 (GROUP, 1) #### A masked pattern was here #### Vertices: @@ -718,8 +718,8 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 2 <- Map 1 (SORT, 1) - Reducer 7 <- Map 6 (SORT, 1) + Reducer 2 <- Map 1 (SORT, 3) + Reducer 7 <- Map 6 (SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -788,7 +788,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 4 <- Map 3 (SORT, 1) + Reducer 4 <- Map 3 (SORT, 3) Reducer 5 <- Reducer 4 (GROUP, 1) #### A masked pattern was here #### Vertices: @@ -924,8 +924,8 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 2 <- Map 1 (SORT, 1) - Reducer 7 <- Map 6 (SORT, 1) + Reducer 2 <- Map 1 (SORT, 3) + Reducer 7 <- Map 6 (SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -994,7 +994,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 4 <- Map 3 (SORT, 1) + Reducer 4 <- Map 3 (SORT, 3) Reducer 5 <- Reducer 4 (GROUP, 1) #### A masked pattern was here #### Vertices: @@ -1130,8 +1130,8 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 5 <- Map 4 (SORT, 1) - Reducer 7 <- Map 6 (SORT, 1) + Reducer 5 <- Map 4 (SORT, 3) + Reducer 7 <- Map 6 (SORT, 3) #### A masked pattern was here #### Vertices: Map 4 @@ -1199,7 +1199,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (SORT, 1) + Reducer 2 <- Map 1 (SORT, 3) Reducer 3 <- Reducer 2 (GROUP, 1) #### A masked pattern was here #### Vertices: @@ -1336,8 +1336,8 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 5 <- Map 4 (SORT, 1) - Reducer 7 <- Map 6 (SORT, 1) + Reducer 5 <- Map 4 (SORT, 3) + Reducer 7 <- Map 6 (SORT, 3) #### A masked pattern was here #### Vertices: Map 4 @@ -1405,7 +1405,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (SORT, 1) + Reducer 2 <- Map 1 (SORT, 3) Reducer 3 <- Reducer 2 (GROUP, 1) #### A masked pattern was here #### Vertices: diff --git a/ql/src/test/results/clientpositive/spark/auto_join32.q.out b/ql/src/test/results/clientpositive/spark/auto_join32.q.out index 9a7a982..fff4d54 100644 --- a/ql/src/test/results/clientpositive/spark/auto_join32.q.out +++ b/ql/src/test/results/clientpositive/spark/auto_join32.q.out @@ -176,7 +176,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -298,7 +298,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -444,7 +444,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 diff --git a/ql/src/test/results/clientpositive/spark/auto_join_without_localtask.q.out b/ql/src/test/results/clientpositive/spark/auto_join_without_localtask.q.out index f5e8f88..26d2165 100644 --- a/ql/src/test/results/clientpositive/spark/auto_join_without_localtask.q.out +++ b/ql/src/test/results/clientpositive/spark/auto_join_without_localtask.q.out @@ -1,8 +1,8 @@ PREHOOK: query: explain -select a.* from src a join src b on a.key=b.key limit 40 +select a.* from src a join src b on a.key=b.key order by key, value limit 40 PREHOOK: type: QUERY POSTHOOK: query: explain -select a.* from src a join src b on a.key=b.key limit 40 +select a.* from src a join src b on a.key=b.key order by key, value limit 40 POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage @@ -12,7 +12,8 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 3 (PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1) + Reducer 3 <- Reducer 2 (SORT, 1) #### A masked pattern was here #### Vertices: Map 1 @@ -28,7 +29,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: key (type: string) Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - Map 3 + Map 4 Map Operator Tree: TableScan alias: a @@ -56,16 +57,26 @@ STAGE PLANS: expressions: _col0 (type: string), _col1 (type: string) outputColumnNames: _col0, _col1 Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE - Limit - Number of rows: 40 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + Reducer 3 + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 40 + Statistics: Num rows: 40 Data size: 400 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false Statistics: Num rows: 40 Data size: 400 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 40 Data size: 400 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -73,11 +84,11 @@ STAGE PLANS: Processor Tree: ListSink -PREHOOK: query: select a.* from src a join src b on a.key=b.key limit 40 +PREHOOK: query: select a.* from src a join src b on a.key=b.key order by key, value limit 40 PREHOOK: type: QUERY PREHOOK: Input: default@src #### A masked pattern was here #### -POSTHOOK: query: select a.* from src a join src b on a.key=b.key limit 40 +POSTHOOK: query: select a.* from src a join src b on a.key=b.key order by key, value limit 40 POSTHOOK: type: QUERY POSTHOOK: Input: default@src #### A masked pattern was here #### @@ -123,10 +134,10 @@ RUN: Stage-1:MAPRED 119 val_119 119 val_119 PREHOOK: query: explain -select a.* from src a join src b on a.key=b.key join src c on a.value=c.value limit 40 +select a.* from src a join src b on a.key=b.key join src c on a.value=c.value order by a.key, a.value limit 40 PREHOOK: type: QUERY POSTHOOK: query: explain -select a.* from src a join src b on a.key=b.key join src c on a.value=c.value limit 40 +select a.* from src a join src b on a.key=b.key join src c on a.value=c.value order by a.key, a.value limit 40 POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage @@ -136,8 +147,9 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 5 (PARTITION-LEVEL SORT, 1) - Reducer 3 <- Map 4 (PARTITION-LEVEL SORT, 1), Reducer 2 (PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 6 (PARTITION-LEVEL SORT, 3) + Reducer 3 <- Map 5 (PARTITION-LEVEL SORT, 3), Reducer 2 (PARTITION-LEVEL SORT, 3) + Reducer 4 <- Reducer 3 (SORT, 1) #### A masked pattern was here #### Vertices: Map 1 @@ -153,7 +165,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: key (type: string) Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - Map 4 + Map 5 Map Operator Tree: TableScan alias: c @@ -166,7 +178,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: value (type: string) Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - Map 5 + Map 6 Map Operator Tree: TableScan alias: a @@ -210,16 +222,26 @@ STAGE PLANS: expressions: _col0 (type: string), _col1 (type: string) outputColumnNames: _col0, _col1 Statistics: Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE - Limit - Number of rows: 40 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Statistics: Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE + Reducer 4 + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 40 + Statistics: Num rows: 40 Data size: 400 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false Statistics: Num rows: 40 Data size: 400 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 40 Data size: 400 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -227,11 +249,11 @@ STAGE PLANS: Processor Tree: ListSink -PREHOOK: query: select a.* from src a join src b on a.key=b.key join src c on a.value=c.value limit 40 +PREHOOK: query: select a.* from src a join src b on a.key=b.key join src c on a.value=c.value order by a.key, a.value limit 40 PREHOOK: type: QUERY PREHOOK: Input: default@src #### A masked pattern was here #### -POSTHOOK: query: select a.* from src a join src b on a.key=b.key join src c on a.value=c.value limit 40 +POSTHOOK: query: select a.* from src a join src b on a.key=b.key join src c on a.value=c.value order by a.key, a.value limit 40 POSTHOOK: type: QUERY POSTHOOK: Input: default@src #### A masked pattern was here #### @@ -290,8 +312,8 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 5 (PARTITION-LEVEL SORT, 1) - Reducer 3 <- Map 4 (PARTITION-LEVEL SORT, 1), Reducer 2 (PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 5 (PARTITION-LEVEL SORT, 3) + Reducer 3 <- Map 4 (PARTITION-LEVEL SORT, 3), Reducer 2 (PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -390,14 +412,6 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@src #### A masked pattern was here #### RUN: Stage-1:MAPRED -103 val_103 -103 val_103 -103 val_103 -103 val_103 -103 val_103 -103 val_103 -103 val_103 -103 val_103 104 val_104 104 val_104 104 val_104 @@ -406,8 +420,6 @@ RUN: Stage-1:MAPRED 104 val_104 104 val_104 104 val_104 -105 val_105 -111 val_111 113 val_113 113 val_113 113 val_113 @@ -416,16 +428,26 @@ RUN: Stage-1:MAPRED 113 val_113 113 val_113 113 val_113 -114 val_114 116 val_116 -118 val_118 -118 val_118 -118 val_118 -118 val_118 -118 val_118 -118 val_118 -118 val_118 -118 val_118 +119 val_119 +119 val_119 +119 val_119 +119 val_119 +119 val_119 +119 val_119 +119 val_119 +119 val_119 +119 val_119 +119 val_119 +119 val_119 +119 val_119 +119 val_119 +119 val_119 +119 val_119 +119 val_119 +119 val_119 +119 val_119 +119 val_119 119 val_119 119 val_119 119 val_119 @@ -441,14 +463,6 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@src #### A masked pattern was here #### RUN: Stage-1:MAPRED -103 val_103 -103 val_103 -103 val_103 -103 val_103 -103 val_103 -103 val_103 -103 val_103 -103 val_103 104 val_104 104 val_104 104 val_104 @@ -457,8 +471,6 @@ RUN: Stage-1:MAPRED 104 val_104 104 val_104 104 val_104 -105 val_105 -111 val_111 113 val_113 113 val_113 113 val_113 @@ -467,16 +479,26 @@ RUN: Stage-1:MAPRED 113 val_113 113 val_113 113 val_113 -114 val_114 116 val_116 -118 val_118 -118 val_118 -118 val_118 -118 val_118 -118 val_118 -118 val_118 -118 val_118 -118 val_118 +119 val_119 +119 val_119 +119 val_119 +119 val_119 +119 val_119 +119 val_119 +119 val_119 +119 val_119 +119 val_119 +119 val_119 +119 val_119 +119 val_119 +119 val_119 +119 val_119 +119 val_119 +119 val_119 +119 val_119 +119 val_119 +119 val_119 119 val_119 119 val_119 119 val_119 diff --git a/ql/src/test/results/clientpositive/spark/auto_smb_mapjoin_14.q.out b/ql/src/test/results/clientpositive/spark/auto_smb_mapjoin_14.q.out index 390af23..c66b49d 100644 --- a/ql/src/test/results/clientpositive/spark/auto_smb_mapjoin_14.q.out +++ b/ql/src/test/results/clientpositive/spark/auto_smb_mapjoin_14.q.out @@ -157,7 +157,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 1) + Reducer 2 <- Map 1 (GROUP, 3) Reducer 3 <- Reducer 2 (GROUP, 1) #### A masked pattern was here #### Vertices: @@ -300,9 +300,9 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 1) - Reducer 5 <- Map 4 (GROUP, 1) - Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 1), Reducer 5 (PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (GROUP, 3) + Reducer 5 <- Map 4 (GROUP, 3) + Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3), Reducer 5 (PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -1639,7 +1639,7 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 1) + Reducer 2 <- Map 1 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 1 diff --git a/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_10.q.out b/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_10.q.out index 445958a..382c340 100644 --- a/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_10.q.out +++ b/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_10.q.out @@ -72,7 +72,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 3 <- Map 6 (PARTITION-LEVEL SORT, 1), Union 2 (PARTITION-LEVEL SORT, 1) + Reducer 3 <- Map 6 (PARTITION-LEVEL SORT, 3), Union 2 (PARTITION-LEVEL SORT, 3) Union 2 <- Map 1 (NONE, 0), Map 5 (NONE, 0) Reducer 4 <- Reducer 3 (GROUP, 1) #### A masked pattern was here #### @@ -227,7 +227,7 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 1) + Reducer 2 <- Map 1 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 1 diff --git a/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_6.q.out b/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_6.q.out index 4562430..731dd1a 100644 --- a/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_6.q.out +++ b/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_6.q.out @@ -96,7 +96,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3) Reducer 3 <- Reducer 2 (GROUP, 1) #### A masked pattern was here #### Vertices: @@ -204,7 +204,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3) Reducer 3 <- Reducer 2 (GROUP, 1) #### A masked pattern was here #### Vertices: @@ -312,7 +312,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3) Reducer 3 <- Reducer 2 (GROUP, 1) #### A masked pattern was here #### Vertices: @@ -416,7 +416,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1), Map 5 (PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3), Map 5 (PARTITION-LEVEL SORT, 3) Reducer 3 <- Reducer 2 (GROUP, 1) #### A masked pattern was here #### Vertices: @@ -613,7 +613,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3) Reducer 3 <- Reducer 2 (GROUP, 1) #### A masked pattern was here #### Vertices: @@ -717,7 +717,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3) Reducer 3 <- Reducer 2 (GROUP, 1) #### A masked pattern was here #### Vertices: @@ -821,7 +821,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1), Map 5 (PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3), Map 5 (PARTITION-LEVEL SORT, 3) Reducer 3 <- Reducer 2 (GROUP, 1) #### A masked pattern was here #### Vertices: @@ -1018,7 +1018,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3) Reducer 3 <- Reducer 2 (GROUP, 1) #### A masked pattern was here #### Vertices: diff --git a/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_9.q.out b/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_9.q.out index d69ab4a..4853aee 100644 --- a/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_9.q.out +++ b/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_9.q.out @@ -151,7 +151,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 1) + Reducer 2 <- Map 1 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -262,7 +262,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 1) + Reducer 2 <- Map 1 (GROUP, 3) Reducer 3 <- Reducer 2 (GROUP, 1) #### A masked pattern was here #### Vertices: @@ -405,9 +405,9 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 1) - Reducer 5 <- Map 4 (GROUP, 1) - Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 1), Reducer 5 (PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (GROUP, 3) + Reducer 5 <- Map 4 (GROUP, 3) + Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3), Reducer 5 (PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -1700,7 +1700,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 1) + Reducer 2 <- Map 1 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -1811,7 +1811,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 1) + Reducer 2 <- Map 1 (GROUP, 3) Reducer 3 <- Reducer 2 (GROUP, 1) #### A masked pattern was here #### Vertices: @@ -1954,9 +1954,9 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 1) - Reducer 5 <- Map 4 (GROUP, 1) - Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 1), Reducer 5 (PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (GROUP, 3) + Reducer 5 <- Map 4 (GROUP, 3) + Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3), Reducer 5 (PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 diff --git a/ql/src/test/results/clientpositive/spark/bucket_map_join_tez1.q.out b/ql/src/test/results/clientpositive/spark/bucket_map_join_tez1.q.out index 2790a18..b0a81e8 100644 --- a/ql/src/test/results/clientpositive/spark/bucket_map_join_tez1.q.out +++ b/ql/src/test/results/clientpositive/spark/bucket_map_join_tez1.q.out @@ -212,7 +212,7 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 3 <- Map 2 (GROUP, 1) + Reducer 3 <- Map 2 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 2 @@ -348,7 +348,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 3 <- Map 2 (GROUP, 1) + Reducer 3 <- Map 2 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 2 @@ -499,7 +499,7 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 3 <- Map 2 (GROUP, 1) + Reducer 3 <- Map 2 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 2 @@ -852,7 +852,7 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 3 <- Map 2 (GROUP, 1) + Reducer 3 <- Map 2 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 2 @@ -965,7 +965,7 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 3 <- Map 2 (GROUP, 1) + Reducer 3 <- Map 2 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 2 diff --git a/ql/src/test/results/clientpositive/spark/bucket_map_join_tez2.q.out b/ql/src/test/results/clientpositive/spark/bucket_map_join_tez2.q.out index b6d05df..6d9cc20 100644 --- a/ql/src/test/results/clientpositive/spark/bucket_map_join_tez2.q.out +++ b/ql/src/test/results/clientpositive/spark/bucket_map_join_tez2.q.out @@ -592,7 +592,7 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 3 <- Map 2 (GROUP, 1) + Reducer 3 <- Map 2 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 2 @@ -696,7 +696,7 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 3 <- Map 2 (GROUP, 1) + Reducer 3 <- Map 2 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 2 diff --git a/ql/src/test/results/clientpositive/spark/count.q.out b/ql/src/test/results/clientpositive/spark/count.q.out index 89d5544..3e4ac5c 100644 --- a/ql/src/test/results/clientpositive/spark/count.q.out +++ b/ql/src/test/results/clientpositive/spark/count.q.out @@ -1,8 +1,10 @@ -PREHOOK: query: create table abcd (a int, b int, c int, d int) +PREHOOK: query: -- SORT_QUERY_RESULTS +create table abcd (a int, b int, c int, d int) PREHOOK: type: CREATETABLE PREHOOK: Output: database:default PREHOOK: Output: default@abcd -POSTHOOK: query: create table abcd (a int, b int, c int, d int) +POSTHOOK: query: -- SORT_QUERY_RESULTS +create table abcd (a int, b int, c int, d int) POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@abcd @@ -22,13 +24,13 @@ POSTHOOK: query: select * from abcd POSTHOOK: type: QUERY POSTHOOK: Input: default@abcd #### A masked pattern was here #### -NULL 35 23 6 +10 100 45 4 +10 100 NULL 5 10 1000 50 1 100 100 10 3 -12 NULL 80 2 -10 100 NULL 5 -10 100 45 4 12 100 75 7 +12 NULL 80 2 +NULL 35 23 6 PREHOOK: query: explain select a, count(distinct b), count(distinct c), sum(d) from abcd group by a PREHOOK: type: QUERY POSTHOOK: query: explain select a, count(distinct b), count(distinct c), sum(d) from abcd group by a @@ -99,10 +101,10 @@ POSTHOOK: query: select a, count(distinct b), count(distinct c), sum(d) from abc POSTHOOK: type: QUERY POSTHOOK: Input: default@abcd #### A masked pattern was here #### -NULL 1 1 6 10 2 2 10 -12 1 2 9 100 1 1 3 +12 1 2 9 +NULL 1 1 6 PREHOOK: query: explain select count(1), count(*), count(a), count(b), count(c), count(d), count(distinct a), count(distinct b), count(distinct c), count(distinct d), count(distinct a,b), count(distinct b,c), count(distinct c,d), count(distinct a,d), count(distinct a,c), count(distinct b,d), count(distinct a,b,c), count(distinct b,c,d), count(distinct a,c,d), count(distinct a,b,d), count(distinct a,b,c,d) from abcd PREHOOK: type: QUERY POSTHOOK: query: explain select count(1), count(*), count(a), count(b), count(c), count(d), count(distinct a), count(distinct b), count(distinct c), count(distinct d), count(distinct a,b), count(distinct b,c), count(distinct c,d), count(distinct a,d), count(distinct a,c), count(distinct b,d), count(distinct a,b,c), count(distinct b,c,d), count(distinct a,c,d), count(distinct a,b,d), count(distinct a,b,c,d) from abcd @@ -184,7 +186,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -236,10 +238,10 @@ POSTHOOK: query: select a, count(distinct b), count(distinct c), sum(d) from abc POSTHOOK: type: QUERY POSTHOOK: Input: default@abcd #### A masked pattern was here #### -NULL 1 1 6 10 2 2 10 -12 1 2 9 100 1 1 3 +12 1 2 9 +NULL 1 1 6 PREHOOK: query: explain select count(1), count(*), count(a), count(b), count(c), count(d), count(distinct a), count(distinct b), count(distinct c), count(distinct d), count(distinct a,b), count(distinct b,c), count(distinct c,d), count(distinct a,d), count(distinct a,c), count(distinct b,d), count(distinct a,b,c), count(distinct b,c,d), count(distinct a,c,d), count(distinct a,b,d), count(distinct a,b,c,d) from abcd PREHOOK: type: QUERY POSTHOOK: query: explain select count(1), count(*), count(a), count(b), count(c), count(d), count(distinct a), count(distinct b), count(distinct c), count(distinct d), count(distinct a,b), count(distinct b,c), count(distinct c,d), count(distinct a,d), count(distinct a,c), count(distinct b,d), count(distinct a,b,c), count(distinct b,c,d), count(distinct a,c,d), count(distinct a,b,d), count(distinct a,b,c,d) from abcd diff --git a/ql/src/test/results/clientpositive/spark/ctas.q.out b/ql/src/test/results/clientpositive/spark/ctas.q.out index 2ee6d9f..15c46ed 100644 --- a/ql/src/test/results/clientpositive/spark/ctas.q.out +++ b/ql/src/test/results/clientpositive/spark/ctas.q.out @@ -176,7 +176,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (SORT, 1) + Reducer 2 <- Map 1 (SORT, 3) Reducer 3 <- Reducer 2 (SORT, 1) #### A masked pattern was here #### Vertices: @@ -320,7 +320,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (SORT, 1) + Reducer 2 <- Map 1 (SORT, 3) Reducer 3 <- Reducer 2 (SORT, 1) #### A masked pattern was here #### Vertices: @@ -529,7 +529,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (SORT, 1) + Reducer 2 <- Map 1 (SORT, 3) Reducer 3 <- Reducer 2 (SORT, 1) #### A masked pattern was here #### Vertices: @@ -716,7 +716,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (SORT, 1) + Reducer 2 <- Map 1 (SORT, 3) Reducer 3 <- Reducer 2 (SORT, 1) #### A masked pattern was here #### Vertices: diff --git a/ql/src/test/results/clientpositive/spark/groupby10.q.out b/ql/src/test/results/clientpositive/spark/groupby10.q.out index 47de08c..31ea336 100644 --- a/ql/src/test/results/clientpositive/spark/groupby10.q.out +++ b/ql/src/test/results/clientpositive/spark/groupby10.q.out @@ -268,10 +268,10 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 5 <- Map 1 (SORT, 1) - Reducer 6 <- Map 1 (SORT, 1) - Reducer 3 <- Reducer 5 (GROUP, 1) - Reducer 4 <- Reducer 6 (GROUP, 1) + Reducer 5 <- Map 1 (SORT, 3) + Reducer 6 <- Map 1 (SORT, 3) + Reducer 3 <- Reducer 5 (GROUP, 3) + Reducer 4 <- Reducer 6 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -483,10 +483,10 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 5 <- Map 1 (SORT, 1) - Reducer 6 <- Map 1 (SORT, 1) - Reducer 3 <- Reducer 5 (GROUP, 1) - Reducer 4 <- Reducer 6 (GROUP, 1) + Reducer 5 <- Map 1 (SORT, 3) + Reducer 6 <- Map 1 (SORT, 3) + Reducer 3 <- Reducer 5 (GROUP, 3) + Reducer 4 <- Reducer 6 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 1 diff --git a/ql/src/test/results/clientpositive/spark/groupby8.q.out b/ql/src/test/results/clientpositive/spark/groupby8.q.out index d943957..c0c0cf2 100644 --- a/ql/src/test/results/clientpositive/spark/groupby8.q.out +++ b/ql/src/test/results/clientpositive/spark/groupby8.q.out @@ -828,10 +828,10 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 5 <- Map 1 (SORT, 1) - Reducer 6 <- Map 1 (SORT, 1) - Reducer 3 <- Reducer 5 (GROUP, 1) - Reducer 4 <- Reducer 6 (GROUP, 1) + Reducer 5 <- Map 1 (SORT, 3) + Reducer 6 <- Map 1 (SORT, 3) + Reducer 3 <- Reducer 5 (GROUP, 3) + Reducer 4 <- Reducer 6 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 1 diff --git a/ql/src/test/results/clientpositive/spark/groupby9.q.out b/ql/src/test/results/clientpositive/spark/groupby9.q.out index 308b777..2707239 100644 --- a/ql/src/test/results/clientpositive/spark/groupby9.q.out +++ b/ql/src/test/results/clientpositive/spark/groupby9.q.out @@ -829,10 +829,10 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 5 <- Map 1 (SORT, 1) - Reducer 6 <- Map 1 (SORT, 1) - Reducer 3 <- Reducer 5 (GROUP, 1) - Reducer 4 <- Reducer 6 (GROUP, 1) + Reducer 5 <- Map 1 (SORT, 3) + Reducer 6 <- Map 1 (SORT, 3) + Reducer 3 <- Reducer 5 (GROUP, 3) + Reducer 4 <- Reducer 6 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -1619,10 +1619,10 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 5 <- Map 1 (SORT, 1) - Reducer 6 <- Map 1 (SORT, 1) - Reducer 3 <- Reducer 5 (GROUP, 1) - Reducer 4 <- Reducer 6 (GROUP, 1) + Reducer 5 <- Map 1 (SORT, 3) + Reducer 6 <- Map 1 (SORT, 3) + Reducer 3 <- Reducer 5 (GROUP, 3) + Reducer 4 <- Reducer 6 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -2409,8 +2409,8 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 2 <- Map 4 (GROUP, 1) - Reducer 3 <- Map 5 (GROUP, 1) + Reducer 2 <- Map 4 (GROUP, 3) + Reducer 3 <- Map 5 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 4 @@ -3196,10 +3196,10 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 5 <- Map 1 (SORT, 1) - Reducer 6 <- Map 1 (SORT, 1) - Reducer 3 <- Reducer 5 (GROUP, 1) - Reducer 4 <- Reducer 6 (GROUP, 1) + Reducer 5 <- Map 1 (SORT, 3) + Reducer 6 <- Map 1 (SORT, 3) + Reducer 3 <- Reducer 5 (GROUP, 3) + Reducer 4 <- Reducer 6 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 1 diff --git a/ql/src/test/results/clientpositive/spark/groupby_cube1.q.out b/ql/src/test/results/clientpositive/spark/groupby_cube1.q.out index 2b6585b..8ed3e67 100644 --- a/ql/src/test/results/clientpositive/spark/groupby_cube1.q.out +++ b/ql/src/test/results/clientpositive/spark/groupby_cube1.q.out @@ -122,7 +122,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -199,8 +199,8 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 1) - Reducer 3 <- Reducer 2 (GROUP PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 3) + Reducer 3 <- Reducer 2 (GROUP PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -304,7 +304,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -404,10 +404,10 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 2 <- Map 6 (GROUP PARTITION-LEVEL SORT, 1) - Reducer 4 <- Map 7 (GROUP PARTITION-LEVEL SORT, 1) - Reducer 3 <- Reducer 2 (GROUP PARTITION-LEVEL SORT, 1) - Reducer 5 <- Reducer 4 (GROUP PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 6 (GROUP PARTITION-LEVEL SORT, 3) + Reducer 4 <- Map 7 (GROUP PARTITION-LEVEL SORT, 3) + Reducer 3 <- Reducer 2 (GROUP PARTITION-LEVEL SORT, 3) + Reducer 5 <- Reducer 4 (GROUP PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 6 diff --git a/ql/src/test/results/clientpositive/spark/groupby_multi_single_reducer3.q.out b/ql/src/test/results/clientpositive/spark/groupby_multi_single_reducer3.q.out index c545d47..fc02ea3 100644 --- a/ql/src/test/results/clientpositive/spark/groupby_multi_single_reducer3.q.out +++ b/ql/src/test/results/clientpositive/spark/groupby_multi_single_reducer3.q.out @@ -225,7 +225,7 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 2 <- Map 1 (SORT, 1) + Reducer 2 <- Map 1 (SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -401,7 +401,7 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 2 <- Map 1 (SORT, 1) + Reducer 2 <- Map 1 (SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -577,7 +577,7 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 2 <- Map 1 (SORT, 1) + Reducer 2 <- Map 1 (SORT, 3) #### A masked pattern was here #### Vertices: Map 1 diff --git a/ql/src/test/results/clientpositive/spark/groupby_position.q.out b/ql/src/test/results/clientpositive/spark/groupby_position.q.out index 8e4e547..6367c43 100644 --- a/ql/src/test/results/clientpositive/spark/groupby_position.q.out +++ b/ql/src/test/results/clientpositive/spark/groupby_position.q.out @@ -238,8 +238,8 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 2 <- Map 4 (GROUP PARTITION-LEVEL SORT, 1) - Reducer 3 <- Map 5 (GROUP PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 4 (GROUP PARTITION-LEVEL SORT, 3) + Reducer 3 <- Map 5 (GROUP PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 4 @@ -438,7 +438,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 1) + Reducer 2 <- Map 1 (GROUP, 3) Reducer 3 <- Reducer 2 (SORT, 1) #### A masked pattern was here #### Vertices: @@ -569,7 +569,7 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 5 <- Map 4 (GROUP, 1) + Reducer 5 <- Map 4 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 4 @@ -618,7 +618,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 3) Reducer 3 <- Reducer 2 (SORT, 1) #### A masked pattern was here #### Vertices: diff --git a/ql/src/test/results/clientpositive/spark/groupby_rollup1.q.out b/ql/src/test/results/clientpositive/spark/groupby_rollup1.q.out index 0563354..8905705 100644 --- a/ql/src/test/results/clientpositive/spark/groupby_rollup1.q.out +++ b/ql/src/test/results/clientpositive/spark/groupby_rollup1.q.out @@ -116,7 +116,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -193,8 +193,8 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 1) - Reducer 3 <- Reducer 2 (GROUP PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 3) + Reducer 3 <- Reducer 2 (GROUP PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -292,7 +292,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -392,10 +392,10 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 2 <- Map 6 (GROUP PARTITION-LEVEL SORT, 1) - Reducer 4 <- Map 7 (GROUP PARTITION-LEVEL SORT, 1) - Reducer 3 <- Reducer 2 (GROUP PARTITION-LEVEL SORT, 1) - Reducer 5 <- Reducer 4 (GROUP PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 6 (GROUP PARTITION-LEVEL SORT, 3) + Reducer 4 <- Map 7 (GROUP PARTITION-LEVEL SORT, 3) + Reducer 3 <- Reducer 2 (GROUP PARTITION-LEVEL SORT, 3) + Reducer 5 <- Reducer 4 (GROUP PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 6 diff --git a/ql/src/test/results/clientpositive/spark/groupby_sort_1_23.q.out b/ql/src/test/results/clientpositive/spark/groupby_sort_1_23.q.out index 9507753..98bb89e 100644 --- a/ql/src/test/results/clientpositive/spark/groupby_sort_1_23.q.out +++ b/ql/src/test/results/clientpositive/spark/groupby_sort_1_23.q.out @@ -297,7 +297,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 1) + Reducer 2 <- Map 1 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -1184,7 +1184,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 1) + Reducer 2 <- Map 1 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -1411,7 +1411,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 1) + Reducer 2 <- Map 1 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -1674,7 +1674,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 1) + Reducer 2 <- Map 1 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -1992,7 +1992,7 @@ STAGE PLANS: columns.types int:int #### A masked pattern was here #### name default.outputtbl1 - numFiles 1 + numFiles 3 numRows 5 rawDataSize 17 serialization.ddl struct outputtbl1 { i32 key, i32 cnt} @@ -2094,7 +2094,7 @@ STAGE PLANS: columns.types int:int #### A masked pattern was here #### name default.outputtbl1 - numFiles 1 + numFiles 3 numRows 5 rawDataSize 17 serialization.ddl struct outputtbl1 { i32 key, i32 cnt} @@ -2179,7 +2179,7 @@ STAGE PLANS: columns.types int:int #### A masked pattern was here #### name default.outputtbl1 - numFiles 1 + numFiles 3 numRows 5 rawDataSize 17 serialization.ddl struct outputtbl1 { i32 key, i32 cnt} @@ -2323,7 +2323,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 4 <- Map 3 (GROUP, 1) + Reducer 4 <- Map 3 (GROUP, 3) Union 2 <- Map 1 (NONE, 0), Reducer 4 (NONE, 0) #### A masked pattern was here #### Vertices: @@ -2886,7 +2886,7 @@ STAGE PLANS: columns.types int:int #### A masked pattern was here #### name default.outputtbl1 - numFiles 2 + numFiles 4 numRows 10 rawDataSize 32 serialization.ddl struct outputtbl1 { i32 key, i32 cnt} @@ -2971,7 +2971,7 @@ STAGE PLANS: columns.types int:int #### A masked pattern was here #### name default.outputtbl1 - numFiles 2 + numFiles 4 numRows 10 rawDataSize 32 serialization.ddl struct outputtbl1 { i32 key, i32 cnt} @@ -3116,7 +3116,7 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 3 <- Map 2 (GROUP, 1) + Reducer 3 <- Map 2 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 2 @@ -3419,7 +3419,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 1) + Reducer 2 <- Map 1 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -3694,7 +3694,7 @@ STAGE PLANS: columns.types int:int:string:int #### A masked pattern was here #### name default.outputtbl4 - numFiles 1 + numFiles 3 numRows 6 rawDataSize 48 serialization.ddl struct outputtbl4 { i32 key1, i32 key2, string key3, i32 cnt} @@ -3777,7 +3777,7 @@ STAGE PLANS: columns.types int:int:string:int #### A masked pattern was here #### name default.outputtbl4 - numFiles 1 + numFiles 3 numRows 6 rawDataSize 48 serialization.ddl struct outputtbl4 { i32 key1, i32 key2, string key3, i32 cnt} diff --git a/ql/src/test/results/clientpositive/spark/groupby_sort_skew_1_23.q.out b/ql/src/test/results/clientpositive/spark/groupby_sort_skew_1_23.q.out index bb2f97f..59db1f1 100644 --- a/ql/src/test/results/clientpositive/spark/groupby_sort_skew_1_23.q.out +++ b/ql/src/test/results/clientpositive/spark/groupby_sort_skew_1_23.q.out @@ -297,8 +297,8 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 1) - Reducer 3 <- Reducer 2 (GROUP, 1) + Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 3) + Reducer 3 <- Reducer 2 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -1202,8 +1202,8 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 1) - Reducer 3 <- Reducer 2 (GROUP, 1) + Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 3) + Reducer 3 <- Reducer 2 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -1447,8 +1447,8 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 1) - Reducer 3 <- Reducer 2 (GROUP, 1) + Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 3) + Reducer 3 <- Reducer 2 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -1728,8 +1728,8 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 1) - Reducer 3 <- Reducer 2 (GROUP, 1) + Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 3) + Reducer 3 <- Reducer 2 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -2064,7 +2064,7 @@ STAGE PLANS: columns.types int:int #### A masked pattern was here #### name default.outputtbl1 - numFiles 1 + numFiles 3 numRows 5 rawDataSize 17 serialization.ddl struct outputtbl1 { i32 key, i32 cnt} @@ -2166,7 +2166,7 @@ STAGE PLANS: columns.types int:int #### A masked pattern was here #### name default.outputtbl1 - numFiles 1 + numFiles 3 numRows 5 rawDataSize 17 serialization.ddl struct outputtbl1 { i32 key, i32 cnt} @@ -2251,7 +2251,7 @@ STAGE PLANS: columns.types int:int #### A masked pattern was here #### name default.outputtbl1 - numFiles 1 + numFiles 3 numRows 5 rawDataSize 17 serialization.ddl struct outputtbl1 { i32 key, i32 cnt} @@ -2395,8 +2395,8 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 4 <- Map 3 (GROUP PARTITION-LEVEL SORT, 1) - Reducer 5 <- Reducer 4 (GROUP, 1) + Reducer 4 <- Map 3 (GROUP PARTITION-LEVEL SORT, 3) + Reducer 5 <- Reducer 4 (GROUP, 3) Union 2 <- Map 1 (NONE, 0), Reducer 5 (NONE, 0) #### A masked pattern was here #### Vertices: @@ -2976,7 +2976,7 @@ STAGE PLANS: columns.types int:int #### A masked pattern was here #### name default.outputtbl1 - numFiles 2 + numFiles 4 numRows 10 rawDataSize 32 serialization.ddl struct outputtbl1 { i32 key, i32 cnt} @@ -3061,7 +3061,7 @@ STAGE PLANS: columns.types int:int #### A masked pattern was here #### name default.outputtbl1 - numFiles 2 + numFiles 4 numRows 10 rawDataSize 32 serialization.ddl struct outputtbl1 { i32 key, i32 cnt} @@ -3206,8 +3206,8 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 3 <- Map 2 (GROUP PARTITION-LEVEL SORT, 1) - Reducer 4 <- Reducer 3 (GROUP, 1) + Reducer 3 <- Map 2 (GROUP PARTITION-LEVEL SORT, 3) + Reducer 4 <- Reducer 3 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 2 @@ -3527,8 +3527,8 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 1) - Reducer 3 <- Reducer 2 (GROUP, 1) + Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 3) + Reducer 3 <- Reducer 2 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -3820,7 +3820,7 @@ STAGE PLANS: columns.types int:int:string:int #### A masked pattern was here #### name default.outputtbl4 - numFiles 1 + numFiles 3 numRows 6 rawDataSize 48 serialization.ddl struct outputtbl4 { i32 key1, i32 key2, string key3, i32 cnt} @@ -3903,7 +3903,7 @@ STAGE PLANS: columns.types int:int:string:int #### A masked pattern was here #### name default.outputtbl4 - numFiles 1 + numFiles 3 numRows 6 rawDataSize 48 serialization.ddl struct outputtbl4 { i32 key1, i32 key2, string key3, i32 cnt} diff --git a/ql/src/test/results/clientpositive/spark/having.q.out b/ql/src/test/results/clientpositive/spark/having.q.out index 5e9f20d..3b80bd1 100644 --- a/ql/src/test/results/clientpositive/spark/having.q.out +++ b/ql/src/test/results/clientpositive/spark/having.q.out @@ -95,7 +95,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 1) + Reducer 2 <- Map 1 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -476,7 +476,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 1) + Reducer 2 <- Map 1 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -748,7 +748,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 1) + Reducer 2 <- Map 1 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -949,7 +949,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 1) + Reducer 2 <- Map 1 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -1221,7 +1221,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 1) + Reducer 2 <- Map 1 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 1 diff --git a/ql/src/test/results/clientpositive/spark/join20.q.out b/ql/src/test/results/clientpositive/spark/join20.q.out index a456fbc..d7c2c7c 100644 --- a/ql/src/test/results/clientpositive/spark/join20.q.out +++ b/ql/src/test/results/clientpositive/spark/join20.q.out @@ -751,7 +751,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 3 <- Map 2 (SORT, 1) + Reducer 3 <- Map 2 (SORT, 3) #### A masked pattern was here #### Vertices: Map 2 diff --git a/ql/src/test/results/clientpositive/spark/join38.q.out b/ql/src/test/results/clientpositive/spark/join38.q.out index d977b4d..a511b03 100644 --- a/ql/src/test/results/clientpositive/spark/join38.q.out +++ b/ql/src/test/results/clientpositive/spark/join38.q.out @@ -79,7 +79,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 3 <- Map 2 (GROUP, 1) + Reducer 3 <- Map 2 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 2 diff --git a/ql/src/test/results/clientpositive/spark/join40.q.out b/ql/src/test/results/clientpositive/spark/join40.q.out index 86c54c1..a4a0dc6 100644 --- a/ql/src/test/results/clientpositive/spark/join40.q.out +++ b/ql/src/test/results/clientpositive/spark/join40.q.out @@ -1843,7 +1843,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 3 <- Map 2 (SORT, 1) + Reducer 3 <- Map 2 (SORT, 3) #### A masked pattern was here #### Vertices: Map 2 @@ -2528,7 +2528,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 3 <- Map 2 (SORT, 1) + Reducer 3 <- Map 2 (SORT, 3) #### A masked pattern was here #### Vertices: Map 2 diff --git a/ql/src/test/results/clientpositive/spark/join_filters_overlap.q.out b/ql/src/test/results/clientpositive/spark/join_filters_overlap.q.out index 07e3601..1a6f157 100644 --- a/ql/src/test/results/clientpositive/spark/join_filters_overlap.q.out +++ b/ql/src/test/results/clientpositive/spark/join_filters_overlap.q.out @@ -1288,7 +1288,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 3 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1), Map 5 (PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 3 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3), Map 5 (PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 diff --git a/ql/src/test/results/clientpositive/spark/limit_pushdown.q.out b/ql/src/test/results/clientpositive/spark/limit_pushdown.q.out index ccc0808..d8443fb 100644 --- a/ql/src/test/results/clientpositive/spark/limit_pushdown.q.out +++ b/ql/src/test/results/clientpositive/spark/limit_pushdown.q.out @@ -173,10 +173,10 @@ POSTHOOK: Input: default@src 98 val_98 98 val_98 PREHOOK: query: explain -select value, sum(key + 1) as sum from src group by value limit 20 +select value, sum(key + 1) as sum from src group by value order by value limit 20 PREHOOK: type: QUERY POSTHOOK: query: explain -select value, sum(key + 1) as sum from src group by value limit 20 +select value, sum(key + 1) as sum from src group by value order by value limit 20 POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage @@ -186,7 +186,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 1) + Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 1) #### A masked pattern was here #### Vertices: Map 1 @@ -240,11 +240,11 @@ STAGE PLANS: Processor Tree: ListSink -PREHOOK: query: select value, sum(key + 1) as sum from src group by value limit 20 +PREHOOK: query: select value, sum(key + 1) as sum from src group by value order by value limit 20 PREHOOK: type: QUERY PREHOOK: Input: default@src #### A masked pattern was here #### -POSTHOOK: query: select value, sum(key + 1) as sum from src group by value limit 20 +POSTHOOK: query: select value, sum(key + 1) as sum from src group by value order by value limit 20 POSTHOOK: type: QUERY POSTHOOK: Input: default@src #### A masked pattern was here #### @@ -382,7 +382,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 1) + Reducer 2 <- Map 1 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -462,10 +462,10 @@ POSTHOOK: Input: default@alltypesorc -16379.0 NULL PREHOOK: query: explain -select ctinyint, count(distinct(cdouble)) from alltypesorc group by ctinyint limit 20 +select ctinyint, count(distinct(cdouble)) from alltypesorc group by ctinyint order by ctinyint limit 20 PREHOOK: type: QUERY POSTHOOK: query: explain -select ctinyint, count(distinct(cdouble)) from alltypesorc group by ctinyint limit 20 +select ctinyint, count(distinct(cdouble)) from alltypesorc group by ctinyint order by ctinyint limit 20 POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage @@ -528,11 +528,11 @@ STAGE PLANS: Processor Tree: ListSink -PREHOOK: query: select ctinyint, count(distinct(cdouble)) from alltypesorc group by ctinyint limit 20 +PREHOOK: query: select ctinyint, count(distinct(cdouble)) from alltypesorc group by ctinyint order by ctinyint limit 20 PREHOOK: type: QUERY PREHOOK: Input: default@alltypesorc #### A masked pattern was here #### -POSTHOOK: query: select ctinyint, count(distinct(cdouble)) from alltypesorc group by ctinyint limit 20 +POSTHOOK: query: select ctinyint, count(distinct(cdouble)) from alltypesorc group by ctinyint order by ctinyint limit 20 POSTHOOK: type: QUERY POSTHOOK: Input: default@alltypesorc #### A masked pattern was here #### @@ -558,11 +558,11 @@ POSTHOOK: Input: default@alltypesorc NULL 2932 PREHOOK: query: -- multi distinct explain -select ctinyint, count(distinct(cstring1)), count(distinct(cstring2)) from alltypesorc group by ctinyint limit 20 +select ctinyint, count(distinct(cstring1)), count(distinct(cstring2)) from alltypesorc group by ctinyint order by ctinyint limit 20 PREHOOK: type: QUERY POSTHOOK: query: -- multi distinct explain -select ctinyint, count(distinct(cstring1)), count(distinct(cstring2)) from alltypesorc group by ctinyint limit 20 +select ctinyint, count(distinct(cstring1)), count(distinct(cstring2)) from alltypesorc group by ctinyint order by ctinyint limit 20 POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage @@ -625,11 +625,11 @@ STAGE PLANS: Processor Tree: ListSink -PREHOOK: query: select ctinyint, count(distinct(cstring1)), count(distinct(cstring2)) from alltypesorc group by ctinyint limit 20 +PREHOOK: query: select ctinyint, count(distinct(cstring1)), count(distinct(cstring2)) from alltypesorc group by ctinyint order by ctinyint limit 20 PREHOOK: type: QUERY PREHOOK: Input: default@alltypesorc #### A masked pattern was here #### -POSTHOOK: query: select ctinyint, count(distinct(cstring1)), count(distinct(cstring2)) from alltypesorc group by ctinyint limit 20 +POSTHOOK: query: select ctinyint, count(distinct(cstring1)), count(distinct(cstring2)) from alltypesorc group by ctinyint order by ctinyint limit 20 POSTHOOK: type: QUERY POSTHOOK: Input: default@alltypesorc #### A masked pattern was here #### @@ -695,7 +695,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 1) + Reducer 2 <- Map 1 (GROUP, 3) Reducer 3 <- Reducer 2 (SORT, 1) #### A masked pattern was here #### Vertices: @@ -870,7 +870,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 4 <- Map 3 (GROUP, 1) + Reducer 4 <- Map 3 (GROUP, 3) Reducer 5 <- Reducer 4 (GROUP, 1) #### A masked pattern was here #### Vertices: @@ -966,11 +966,11 @@ STAGE PLANS: PREHOOK: query: -- map aggregation disabled explain -select value, sum(key) as sum from src group by value limit 20 +select value, sum(key) as sum from src group by value order by value limit 20 PREHOOK: type: QUERY POSTHOOK: query: -- map aggregation disabled explain -select value, sum(key) as sum from src group by value limit 20 +select value, sum(key) as sum from src group by value order by value limit 20 POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage @@ -980,7 +980,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 1) + Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 1) #### A masked pattern was here #### Vertices: Map 1 @@ -1028,20 +1028,23 @@ STAGE PLANS: Processor Tree: ListSink -PREHOOK: query: select value, sum(key) as sum from src group by value limit 20 +PREHOOK: query: select value, sum(key) as sum from src group by value order by value limit 20 PREHOOK: type: QUERY PREHOOK: Input: default@src #### A masked pattern was here #### -POSTHOOK: query: select value, sum(key) as sum from src group by value limit 20 +POSTHOOK: query: select value, sum(key) as sum from src group by value order by value limit 20 POSTHOOK: type: QUERY POSTHOOK: Input: default@src #### A masked pattern was here #### +val_0 0.0 val_10 10.0 val_100 200.0 val_103 206.0 +val_104 208.0 val_105 105.0 val_11 11.0 val_111 111.0 +val_113 226.0 val_114 114.0 val_116 116.0 val_118 236.0 @@ -1049,13 +1052,10 @@ val_119 357.0 val_12 24.0 val_120 240.0 val_125 250.0 +val_126 126.0 val_128 384.0 +val_129 258.0 val_131 131.0 -val_137 137.0 -val_138 414.0 -val_15 15.0 -val_174 174.0 -val_193 193.0 PREHOOK: query: -- flush for order-by explain select key,value,value,value,value,value,value,value,value from src order by key limit 100 @@ -1237,7 +1237,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 1) + Reducer 2 <- Map 1 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -1293,103 +1293,103 @@ POSTHOOK: query: select sum(key) as sum from src group by concat(key,value,value POSTHOOK: type: QUERY POSTHOOK: Input: default@src #### A masked pattern was here #### -0.0 10.0 105.0 -11.0 -111.0 +1107.0 114.0 +1227.0 +1251.0 126.0 -131.0 +1314.0 133.0 136.0 143.0 145.0 -150.0 +1494.0 +15.0 153.0 155.0 -156.0 -157.0 -158.0 -160.0 -162.0 163.0 166.0 17.0 -170.0 -177.0 -178.0 -180.0 -181.0 183.0 -186.0 -189.0 -19.0 -192.0 +1872.0 +194.0 194.0 -196.0 -20.0 -200.0 +1956.0 201.0 202.0 -206.0 +208.0 +210.0 214.0 222.0 226.0 -226.0 -228.0 -24.0 -24.0 -240.0 -242.0 +2345.0 +241.0 250.0 -258.0 +252.0 +262.0 +263.0 268.0 -272.0 -273.0 -274.0 -280.0 -282.0 -298.0 -304.0 -309.0 -316.0 -327.0 +283.0 +291.0 +292.0 +30.0 +310.0 328.0 +33.0 330.0 -348.0 +339.0 +341.0 348.0 350.0 -352.0 -357.0 -358.0 -36.0 -382.0 -384.0 +351.0 +360.0 +379.0 +389.0 390.0 -394.0 -399.0 -400.0 +4.0 406.0 -417.0 +41.0 418.0 +419.0 426.0 -430.0 -432.0 +437.0 442.0 446.0 448.0 -458.0 -459.0 +448.0 +449.0 +457.0 466.0 -469.0 -474.0 -478.0 -489.0 -552.0 -561.0 +467.0 +477.0 +479.0 +484.0 +487.0 +496.0 +497.0 +53.0 +544.0 +562.0 +564.0 +57.0 579.0 -597.0 -622.0 -624.0 -636.0 +618.0 +642.0 +65.0 +662.0 +69.0 +74.0 +77.0 +798.0 +819.0 +82.0 +84.0 +85.0 +858.0 +878.0 +916.0 +918.0 +933.0 +956.0 diff --git a/ql/src/test/results/clientpositive/spark/mapjoin_distinct.q.out b/ql/src/test/results/clientpositive/spark/mapjoin_distinct.q.out index 07b0152..2096a8a 100644 --- a/ql/src/test/results/clientpositive/spark/mapjoin_distinct.q.out +++ b/ql/src/test/results/clientpositive/spark/mapjoin_distinct.q.out @@ -190,7 +190,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 3 <- Map 2 (GROUP, 1) + Reducer 3 <- Map 2 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 2 @@ -326,8 +326,8 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 3 <- Map 2 (GROUP PARTITION-LEVEL SORT, 1) - Reducer 4 <- Reducer 3 (GROUP, 1) + Reducer 3 <- Map 2 (GROUP PARTITION-LEVEL SORT, 3) + Reducer 4 <- Reducer 3 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 2 @@ -470,7 +470,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 3 <- Map 2 (GROUP, 1) + Reducer 3 <- Map 2 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 2 diff --git a/ql/src/test/results/clientpositive/spark/mapjoin_filter_on_outerjoin.q.out b/ql/src/test/results/clientpositive/spark/mapjoin_filter_on_outerjoin.q.out index 287f19d..b5e58d5 100644 --- a/ql/src/test/results/clientpositive/spark/mapjoin_filter_on_outerjoin.q.out +++ b/ql/src/test/results/clientpositive/spark/mapjoin_filter_on_outerjoin.q.out @@ -112,7 +112,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 3 <- Map 2 (SORT, 1) + Reducer 3 <- Map 2 (SORT, 3) #### A masked pattern was here #### Vertices: Map 2 @@ -285,7 +285,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 3 <- Map 2 (SORT, 1) + Reducer 3 <- Map 2 (SORT, 3) #### A masked pattern was here #### Vertices: Map 2 diff --git a/ql/src/test/results/clientpositive/spark/mapjoin_test_outer.q.out b/ql/src/test/results/clientpositive/spark/mapjoin_test_outer.q.out index f458227..4bc10b5 100644 --- a/ql/src/test/results/clientpositive/spark/mapjoin_test_outer.q.out +++ b/ql/src/test/results/clientpositive/spark/mapjoin_test_outer.q.out @@ -289,7 +289,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (SORT, 1) + Reducer 2 <- Map 1 (SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -1131,7 +1131,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (SORT, 1) + Reducer 2 <- Map 1 (SORT, 3) #### A masked pattern was here #### Vertices: Map 1 diff --git a/ql/src/test/results/clientpositive/spark/mapreduce2.q.out b/ql/src/test/results/clientpositive/spark/mapreduce2.q.out index e85410b..7956fd0 100644 --- a/ql/src/test/results/clientpositive/spark/mapreduce2.q.out +++ b/ql/src/test/results/clientpositive/spark/mapreduce2.q.out @@ -101,11 +101,11 @@ POSTHOOK: Lineage: dest1.key SCRIPT [(src)src.FieldSchema(name:key, type:string, POSTHOOK: Lineage: dest1.one SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: dest1.ten SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: dest1.value SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ] -PREHOOK: query: SELECT * FROM (SELECT dest1.* FROM dest1 DISTRIBUTE BY key SORT BY key, ten, one, value) T +PREHOOK: query: SELECT * FROM (SELECT dest1.* FROM dest1 DISTRIBUTE BY key SORT BY key, ten, one, value) T ORDER BY key PREHOOK: type: QUERY PREHOOK: Input: default@dest1 #### A masked pattern was here #### -POSTHOOK: query: SELECT * FROM (SELECT dest1.* FROM dest1 DISTRIBUTE BY key SORT BY key, ten, one, value) T +POSTHOOK: query: SELECT * FROM (SELECT dest1.* FROM dest1 DISTRIBUTE BY key SORT BY key, ten, one, value) T ORDER BY key POSTHOOK: type: QUERY POSTHOOK: Input: default@dest1 #### A masked pattern was here #### diff --git a/ql/src/test/results/clientpositive/spark/multi_insert.q.out b/ql/src/test/results/clientpositive/spark/multi_insert.q.out index ebb3616..941c430 100644 --- a/ql/src/test/results/clientpositive/spark/multi_insert.q.out +++ b/ql/src/test/results/clientpositive/spark/multi_insert.q.out @@ -587,7 +587,7 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 2 <- Map 1 (SORT, 1) + Reducer 2 <- Map 1 (SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -745,7 +745,7 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 2 <- Map 1 (SORT, 1) + Reducer 2 <- Map 1 (SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -903,7 +903,7 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 2 <- Map 1 (SORT, 1) + Reducer 2 <- Map 1 (SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -1061,7 +1061,7 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 2 <- Map 1 (SORT, 1) + Reducer 2 <- Map 1 (SORT, 3) #### A masked pattern was here #### Vertices: Map 1 diff --git a/ql/src/test/results/clientpositive/spark/multi_insert_gby.q.out b/ql/src/test/results/clientpositive/spark/multi_insert_gby.q.out index 8d32943..5bbfe19 100644 --- a/ql/src/test/results/clientpositive/spark/multi_insert_gby.q.out +++ b/ql/src/test/results/clientpositive/spark/multi_insert_gby.q.out @@ -234,7 +234,7 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 2 <- Map 1 (SORT, 1) + Reducer 2 <- Map 1 (SORT, 3) #### A masked pattern was here #### Vertices: Map 1 diff --git a/ql/src/test/results/clientpositive/spark/multi_insert_gby3.q.out b/ql/src/test/results/clientpositive/spark/multi_insert_gby3.q.out index a8f1818..fe29a5a 100644 --- a/ql/src/test/results/clientpositive/spark/multi_insert_gby3.q.out +++ b/ql/src/test/results/clientpositive/spark/multi_insert_gby3.q.out @@ -1,8 +1,10 @@ -PREHOOK: query: create table e1 (key string, keyD double) +PREHOOK: query: -- SORT_QUERY_RESULTS +create table e1 (key string, keyD double) PREHOOK: type: CREATETABLE PREHOOK: Output: database:default PREHOOK: Output: default@e1 -POSTHOOK: query: create table e1 (key string, keyD double) +POSTHOOK: query: -- SORT_QUERY_RESULTS +create table e1 (key string, keyD double) POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@e1 @@ -1605,8 +1607,8 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 2 <- Map 4 (GROUP PARTITION-LEVEL SORT, 1) - Reducer 3 <- Map 5 (GROUP PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 4 (GROUP PARTITION-LEVEL SORT, 3) + Reducer 3 <- Map 5 (GROUP PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 4 @@ -1748,7 +1750,7 @@ STAGE PLANS: Spark Edges: Reducer 2 <- Map 1 (SORT, 1) - Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 1) + Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 diff --git a/ql/src/test/results/clientpositive/spark/multi_insert_lateral_view.q.out b/ql/src/test/results/clientpositive/spark/multi_insert_lateral_view.q.out index 08ad9a0..9c19cdd 100644 --- a/ql/src/test/results/clientpositive/spark/multi_insert_lateral_view.q.out +++ b/ql/src/test/results/clientpositive/spark/multi_insert_lateral_view.q.out @@ -291,8 +291,8 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 2 <- Map 4 (GROUP, 1) - Reducer 3 <- Map 5 (GROUP, 1) + Reducer 2 <- Map 4 (GROUP, 3) + Reducer 3 <- Map 5 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 4 @@ -563,8 +563,8 @@ STAGE PLANS: Stage: Stage-3 Spark Edges: - Reducer 2 <- Map 4 (GROUP, 1) - Reducer 3 <- Map 5 (SORT, 1) + Reducer 2 <- Map 4 (GROUP, 3) + Reducer 3 <- Map 5 (SORT, 3) #### A masked pattern was here #### Vertices: Map 4 @@ -854,9 +854,9 @@ STAGE PLANS: Stage: Stage-3 Spark Edges: - Reducer 2 <- Map 5 (GROUP PARTITION-LEVEL SORT, 1) - Reducer 3 <- Map 6 (GROUP PARTITION-LEVEL SORT, 1) - Reducer 4 <- Map 7 (GROUP PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 5 (GROUP PARTITION-LEVEL SORT, 3) + Reducer 3 <- Map 6 (GROUP PARTITION-LEVEL SORT, 3) + Reducer 4 <- Map 7 (GROUP PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 5 @@ -1224,9 +1224,9 @@ STAGE PLANS: Stage: Stage-4 Spark Edges: - Reducer 2 <- Map 5 (GROUP PARTITION-LEVEL SORT, 1) - Reducer 3 <- Map 6 (GROUP PARTITION-LEVEL SORT, 1) - Reducer 4 <- Map 7 (PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 5 (GROUP PARTITION-LEVEL SORT, 3) + Reducer 3 <- Map 6 (GROUP PARTITION-LEVEL SORT, 3) + Reducer 4 <- Map 7 (PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 5 diff --git a/ql/src/test/results/clientpositive/spark/multi_insert_move_tasks_share_dependencies.q.out b/ql/src/test/results/clientpositive/spark/multi_insert_move_tasks_share_dependencies.q.out index 6a00b3b..ea17fa1 100644 --- a/ql/src/test/results/clientpositive/spark/multi_insert_move_tasks_share_dependencies.q.out +++ b/ql/src/test/results/clientpositive/spark/multi_insert_move_tasks_share_dependencies.q.out @@ -604,7 +604,7 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 2 <- Map 1 (SORT, 1) + Reducer 2 <- Map 1 (SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -766,7 +766,7 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 2 <- Map 1 (SORT, 1) + Reducer 2 <- Map 1 (SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -928,7 +928,7 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 2 <- Map 1 (SORT, 1) + Reducer 2 <- Map 1 (SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -1090,7 +1090,7 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 2 <- Map 1 (SORT, 1) + Reducer 2 <- Map 1 (SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -2417,10 +2417,10 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 5 <- Map 1 (SORT, 1) - Reducer 6 <- Map 1 (SORT, 1) - Reducer 3 <- Reducer 5 (SORT, 1) - Reducer 4 <- Reducer 6 (SORT, 1) + Reducer 5 <- Map 1 (SORT, 3) + Reducer 6 <- Map 1 (SORT, 3) + Reducer 3 <- Reducer 5 (SORT, 3) + Reducer 4 <- Reducer 6 (SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -2556,10 +2556,10 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 5 <- Map 1 (SORT, 1) - Reducer 6 <- Map 1 (SORT, 1) - Reducer 3 <- Reducer 5 (SORT, 1) - Reducer 4 <- Reducer 6 (SORT, 1) + Reducer 5 <- Map 1 (SORT, 3) + Reducer 6 <- Map 1 (SORT, 3) + Reducer 3 <- Reducer 5 (SORT, 3) + Reducer 4 <- Reducer 6 (SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -2695,10 +2695,10 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 5 <- Map 1 (SORT, 1) - Reducer 6 <- Map 1 (SORT, 1) - Reducer 3 <- Reducer 5 (SORT, 1) - Reducer 4 <- Reducer 6 (SORT, 1) + Reducer 5 <- Map 1 (SORT, 3) + Reducer 6 <- Map 1 (SORT, 3) + Reducer 3 <- Reducer 5 (SORT, 3) + Reducer 4 <- Reducer 6 (SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -2834,10 +2834,10 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 5 <- Map 1 (SORT, 1) - Reducer 6 <- Map 1 (SORT, 1) - Reducer 3 <- Reducer 5 (SORT, 1) - Reducer 4 <- Reducer 6 (SORT, 1) + Reducer 5 <- Map 1 (SORT, 3) + Reducer 6 <- Map 1 (SORT, 3) + Reducer 3 <- Reducer 5 (SORT, 3) + Reducer 4 <- Reducer 6 (SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -2981,10 +2981,10 @@ STAGE PLANS: Stage: Stage-4 Spark Edges: - Reducer 5 <- Map 1 (SORT, 1) - Reducer 6 <- Map 1 (SORT, 1) - Reducer 3 <- Reducer 5 (SORT, 1) - Reducer 4 <- Reducer 6 (SORT, 1) + Reducer 5 <- Map 1 (SORT, 3) + Reducer 6 <- Map 1 (SORT, 3) + Reducer 3 <- Reducer 5 (SORT, 3) + Reducer 4 <- Reducer 6 (SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -3231,10 +3231,10 @@ STAGE PLANS: Stage: Stage-4 Spark Edges: - Reducer 5 <- Map 1 (SORT, 1) - Reducer 6 <- Map 1 (SORT, 1) - Reducer 3 <- Reducer 5 (SORT, 1) - Reducer 4 <- Reducer 6 (SORT, 1) + Reducer 5 <- Map 1 (SORT, 3) + Reducer 6 <- Map 1 (SORT, 3) + Reducer 3 <- Reducer 5 (SORT, 3) + Reducer 4 <- Reducer 6 (SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -3481,10 +3481,10 @@ STAGE PLANS: Stage: Stage-4 Spark Edges: - Reducer 5 <- Map 1 (SORT, 1) - Reducer 6 <- Map 1 (SORT, 1) - Reducer 3 <- Reducer 5 (SORT, 1) - Reducer 4 <- Reducer 6 (SORT, 1) + Reducer 5 <- Map 1 (SORT, 3) + Reducer 6 <- Map 1 (SORT, 3) + Reducer 3 <- Reducer 5 (SORT, 3) + Reducer 4 <- Reducer 6 (SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -3731,10 +3731,10 @@ STAGE PLANS: Stage: Stage-4 Spark Edges: - Reducer 5 <- Map 1 (SORT, 1) - Reducer 6 <- Map 1 (SORT, 1) - Reducer 3 <- Reducer 5 (SORT, 1) - Reducer 4 <- Reducer 6 (SORT, 1) + Reducer 5 <- Map 1 (SORT, 3) + Reducer 6 <- Map 1 (SORT, 3) + Reducer 3 <- Reducer 5 (SORT, 3) + Reducer 4 <- Reducer 6 (SORT, 3) #### A masked pattern was here #### Vertices: Map 1 diff --git a/ql/src/test/results/clientpositive/spark/multi_join_union.q.out b/ql/src/test/results/clientpositive/spark/multi_join_union.q.out index 3bec983..e0fe067 100644 --- a/ql/src/test/results/clientpositive/spark/multi_join_union.q.out +++ b/ql/src/test/results/clientpositive/spark/multi_join_union.q.out @@ -1,9 +1,13 @@ -PREHOOK: query: CREATE TABLE src11 as SELECT * FROM src +PREHOOK: query: -- SORT_QUERY_RESULTS + +CREATE TABLE src11 as SELECT * FROM src PREHOOK: type: CREATETABLE_AS_SELECT PREHOOK: Input: default@src PREHOOK: Output: database:default PREHOOK: Output: default@src11 -POSTHOOK: query: CREATE TABLE src11 as SELECT * FROM src +POSTHOOK: query: -- SORT_QUERY_RESULTS + +CREATE TABLE src11 as SELECT * FROM src POSTHOOK: type: CREATETABLE_AS_SELECT POSTHOOK: Input: default@src POSTHOOK: Output: database:default @@ -79,7 +83,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 4 <- Map 5 (PARTITION-LEVEL SORT, 1), Union 3 (PARTITION-LEVEL SORT, 1) + Reducer 4 <- Map 5 (PARTITION-LEVEL SORT, 3), Union 3 (PARTITION-LEVEL SORT, 3) Union 3 <- Map 2 (NONE, 0), Map 6 (NONE, 0) #### A masked pattern was here #### Vertices: diff --git a/ql/src/test/results/clientpositive/spark/ppd_join4.q.out b/ql/src/test/results/clientpositive/spark/ppd_join4.q.out index 9a8dcf0..71439cd 100644 --- a/ql/src/test/results/clientpositive/spark/ppd_join4.q.out +++ b/ql/src/test/results/clientpositive/spark/ppd_join4.q.out @@ -75,7 +75,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 3 <- Map 2 (SORT, 1) + Reducer 3 <- Map 2 (SORT, 3) #### A masked pattern was here #### Vertices: Map 2 diff --git a/ql/src/test/results/clientpositive/spark/ppd_join_filter.q.out b/ql/src/test/results/clientpositive/spark/ppd_join_filter.q.out index eee3bc7..29acb9b 100644 --- a/ql/src/test/results/clientpositive/spark/ppd_join_filter.q.out +++ b/ql/src/test/results/clientpositive/spark/ppd_join_filter.q.out @@ -511,7 +511,7 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 3 <- Map 2 (GROUP, 1) + Reducer 3 <- Map 2 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 2 @@ -894,7 +894,7 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 3 <- Map 2 (GROUP, 1) + Reducer 3 <- Map 2 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 2 @@ -1273,7 +1273,7 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 3 <- Map 2 (GROUP, 1) + Reducer 3 <- Map 2 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 2 diff --git a/ql/src/test/results/clientpositive/spark/ppd_outer_join3.q.out b/ql/src/test/results/clientpositive/spark/ppd_outer_join3.q.out index 0978c4f..bbd671b 100644 --- a/ql/src/test/results/clientpositive/spark/ppd_outer_join3.q.out +++ b/ql/src/test/results/clientpositive/spark/ppd_outer_join3.q.out @@ -1,4 +1,6 @@ -PREHOOK: query: EXPLAIN +PREHOOK: query: -- SORT_QUERY_RESULTS + +EXPLAIN FROM src a FULL OUTER JOIN @@ -7,7 +9,9 @@ PREHOOK: query: EXPLAIN SELECT a.key, a.value, b.key, b.value WHERE a.key > '10' AND a.key < '20' AND b.key > '15' AND b.key < '25' PREHOOK: type: QUERY -POSTHOOK: query: EXPLAIN +POSTHOOK: query: -- SORT_QUERY_RESULTS + +EXPLAIN FROM src a FULL OUTER JOIN @@ -252,7 +256,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 3 (PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 3 (PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 diff --git a/ql/src/test/results/clientpositive/spark/ppd_transform.q.out b/ql/src/test/results/clientpositive/spark/ppd_transform.q.out index b8c32a8..6ee9f72 100644 --- a/ql/src/test/results/clientpositive/spark/ppd_transform.q.out +++ b/ql/src/test/results/clientpositive/spark/ppd_transform.q.out @@ -206,7 +206,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (SORT, 1) + Reducer 2 <- Map 1 (SORT, 3) #### A masked pattern was here #### Vertices: Map 1 diff --git a/ql/src/test/results/clientpositive/spark/ptf_decimal.q.out b/ql/src/test/results/clientpositive/spark/ptf_decimal.q.out index fd51b53..8494e97 100644 --- a/ql/src/test/results/clientpositive/spark/ptf_decimal.q.out +++ b/ql/src/test/results/clientpositive/spark/ptf_decimal.q.out @@ -1,4 +1,6 @@ -PREHOOK: query: -- 1. aggregate functions with decimal type +PREHOOK: query: -- SORT_QUERY_RESULTS + +-- 1. aggregate functions with decimal type select p_mfgr, p_retailprice, lead(p_retailprice) over (partition by p_mfgr ORDER BY p_name) as c1, @@ -9,7 +11,9 @@ from part PREHOOK: type: QUERY PREHOOK: Input: default@part #### A masked pattern was here #### -POSTHOOK: query: -- 1. aggregate functions with decimal type +POSTHOOK: query: -- SORT_QUERY_RESULTS + +-- 1. aggregate functions with decimal type select p_mfgr, p_retailprice, lead(p_retailprice) over (partition by p_mfgr ORDER BY p_name) as c1, @@ -22,30 +26,30 @@ POSTHOOK: Input: default@part #### A masked pattern was here #### Manufacturer#1 1173.15 1173.15 NULL 1173.15 1173.15 Manufacturer#1 1173.15 1753.76 1173.15 1173.15 1173.15 -Manufacturer#1 1753.76 1602.59 1173.15 1173.15 1753.76 -Manufacturer#1 1602.59 1414.42 1753.76 1173.15 1602.59 Manufacturer#1 1414.42 1632.66 1602.59 1173.15 1414.42 +Manufacturer#1 1602.59 1414.42 1753.76 1173.15 1602.59 Manufacturer#1 1632.66 NULL 1414.42 1173.15 1632.66 +Manufacturer#1 1753.76 1602.59 1173.15 1173.15 1753.76 Manufacturer#2 1690.68 1800.7 NULL 1690.68 1690.68 -Manufacturer#2 1800.7 2031.98 1690.68 1690.68 1800.7 -Manufacturer#2 2031.98 1698.66 1800.7 1690.68 2031.98 Manufacturer#2 1698.66 1701.6 2031.98 1690.68 1698.66 Manufacturer#2 1701.6 NULL 1698.66 1690.68 1701.6 -Manufacturer#3 1671.68 1190.27 NULL 1671.68 1671.68 +Manufacturer#2 1800.7 2031.98 1690.68 1690.68 1800.7 +Manufacturer#2 2031.98 1698.66 1800.7 1690.68 2031.98 Manufacturer#3 1190.27 1410.39 1671.68 1671.68 1190.27 +Manufacturer#3 1337.29 NULL 1922.98 1671.68 1337.29 Manufacturer#3 1410.39 1922.98 1190.27 1671.68 1410.39 +Manufacturer#3 1671.68 1190.27 NULL 1671.68 1671.68 Manufacturer#3 1922.98 1337.29 1410.39 1671.68 1922.98 -Manufacturer#3 1337.29 NULL 1922.98 1671.68 1337.29 -Manufacturer#4 1620.67 1375.42 NULL 1620.67 1620.67 -Manufacturer#4 1375.42 1206.26 1620.67 1620.67 1375.42 Manufacturer#4 1206.26 1844.92 1375.42 1620.67 1206.26 -Manufacturer#4 1844.92 1290.35 1206.26 1620.67 1844.92 Manufacturer#4 1290.35 NULL 1844.92 1620.67 1290.35 -Manufacturer#5 1789.69 1611.66 NULL 1789.69 1789.69 -Manufacturer#5 1611.66 1788.73 1789.69 1789.69 1611.66 -Manufacturer#5 1788.73 1018.1 1611.66 1789.69 1788.73 +Manufacturer#4 1375.42 1206.26 1620.67 1620.67 1375.42 +Manufacturer#4 1620.67 1375.42 NULL 1620.67 1620.67 +Manufacturer#4 1844.92 1290.35 1206.26 1620.67 1844.92 Manufacturer#5 1018.1 1464.48 1788.73 1789.69 1018.1 Manufacturer#5 1464.48 NULL 1018.1 1789.69 1464.48 +Manufacturer#5 1611.66 1788.73 1789.69 1789.69 1611.66 +Manufacturer#5 1788.73 1018.1 1611.66 1789.69 1788.73 +Manufacturer#5 1789.69 1611.66 NULL 1789.69 1789.69 PREHOOK: query: -- 2. ranking functions with decimal type select p_mfgr, p_retailprice, @@ -114,32 +118,32 @@ from part POSTHOOK: type: QUERY POSTHOOK: Input: default@part #### A masked pattern was here #### -Manufacturer#1 1753.76 NULL -Manufacturer#1 1632.66 1753.76 -Manufacturer#1 1602.59 1632.66 -Manufacturer#1 1414.42 1602.59 -Manufacturer#1 1173.15 1414.42 Manufacturer#1 1173.15 1173.15 -Manufacturer#2 2031.98 NULL -Manufacturer#2 1800.7 2031.98 -Manufacturer#2 1701.6 1800.7 -Manufacturer#2 1698.66 1701.6 +Manufacturer#1 1173.15 1414.42 +Manufacturer#1 1414.42 1602.59 +Manufacturer#1 1602.59 1632.66 +Manufacturer#1 1632.66 1753.76 +Manufacturer#1 1753.76 NULL Manufacturer#2 1690.68 1698.66 -Manufacturer#3 1922.98 NULL -Manufacturer#3 1671.68 1922.98 -Manufacturer#3 1410.39 1671.68 -Manufacturer#3 1337.29 1410.39 +Manufacturer#2 1698.66 1701.6 +Manufacturer#2 1701.6 1800.7 +Manufacturer#2 1800.7 2031.98 +Manufacturer#2 2031.98 NULL Manufacturer#3 1190.27 1337.29 -Manufacturer#4 1844.92 NULL -Manufacturer#4 1620.67 1844.92 -Manufacturer#4 1375.42 1620.67 -Manufacturer#4 1290.35 1375.42 +Manufacturer#3 1337.29 1410.39 +Manufacturer#3 1410.39 1671.68 +Manufacturer#3 1671.68 1922.98 +Manufacturer#3 1922.98 NULL Manufacturer#4 1206.26 1290.35 -Manufacturer#5 1789.69 NULL -Manufacturer#5 1788.73 1789.69 -Manufacturer#5 1611.66 1788.73 -Manufacturer#5 1464.48 1611.66 +Manufacturer#4 1290.35 1375.42 +Manufacturer#4 1375.42 1620.67 +Manufacturer#4 1620.67 1844.92 +Manufacturer#4 1844.92 NULL Manufacturer#5 1018.1 1464.48 +Manufacturer#5 1464.48 1611.66 +Manufacturer#5 1611.66 1788.73 +Manufacturer#5 1788.73 1789.69 +Manufacturer#5 1789.69 NULL PREHOOK: query: -- 4. partition by decimal select p_mfgr, p_retailprice, @@ -156,29 +160,29 @@ from part POSTHOOK: type: QUERY POSTHOOK: Input: default@part #### A masked pattern was here #### -Manufacturer#5 1018.1 NULL -Manufacturer#1 1173.15 NULL Manufacturer#1 1173.15 1173.15 -Manufacturer#3 1190.27 NULL -Manufacturer#4 1206.26 NULL -Manufacturer#4 1290.35 NULL -Manufacturer#3 1337.29 NULL -Manufacturer#4 1375.42 NULL -Manufacturer#3 1410.39 NULL +Manufacturer#1 1173.15 NULL Manufacturer#1 1414.42 NULL -Manufacturer#5 1464.48 NULL Manufacturer#1 1602.59 NULL -Manufacturer#5 1611.66 NULL -Manufacturer#4 1620.67 NULL Manufacturer#1 1632.66 NULL -Manufacturer#3 1671.68 NULL +Manufacturer#1 1753.76 NULL Manufacturer#2 1690.68 NULL Manufacturer#2 1698.66 NULL Manufacturer#2 1701.6 NULL -Manufacturer#1 1753.76 NULL -Manufacturer#5 1788.73 NULL -Manufacturer#5 1789.69 NULL Manufacturer#2 1800.7 NULL -Manufacturer#4 1844.92 NULL -Manufacturer#3 1922.98 NULL Manufacturer#2 2031.98 NULL +Manufacturer#3 1190.27 NULL +Manufacturer#3 1337.29 NULL +Manufacturer#3 1410.39 NULL +Manufacturer#3 1671.68 NULL +Manufacturer#3 1922.98 NULL +Manufacturer#4 1206.26 NULL +Manufacturer#4 1290.35 NULL +Manufacturer#4 1375.42 NULL +Manufacturer#4 1620.67 NULL +Manufacturer#4 1844.92 NULL +Manufacturer#5 1018.1 NULL +Manufacturer#5 1464.48 NULL +Manufacturer#5 1611.66 NULL +Manufacturer#5 1788.73 NULL +Manufacturer#5 1789.69 NULL diff --git a/ql/src/test/results/clientpositive/spark/ptf_general_queries.q.out b/ql/src/test/results/clientpositive/spark/ptf_general_queries.q.out index 25068da..a3d6194 100644 --- a/ql/src/test/results/clientpositive/spark/ptf_general_queries.q.out +++ b/ql/src/test/results/clientpositive/spark/ptf_general_queries.q.out @@ -1,4 +1,6 @@ -PREHOOK: query: -- 1. testNoPTFNoWindowing +PREHOOK: query: -- SORT_QUERY_RESULTS + +-- 1. testNoPTFNoWindowing select p_mfgr, p_name, p_size from part distribute by p_mfgr @@ -6,7 +8,9 @@ sort by p_name PREHOOK: type: QUERY PREHOOK: Input: default@part #### A masked pattern was here #### -POSTHOOK: query: -- 1. testNoPTFNoWindowing +POSTHOOK: query: -- SORT_QUERY_RESULTS + +-- 1. testNoPTFNoWindowing select p_mfgr, p_name, p_size from part distribute by p_mfgr @@ -14,31 +18,31 @@ sort by p_name POSTHOOK: type: QUERY POSTHOOK: Input: default@part #### A masked pattern was here #### -Manufacturer#5 almond antique blue firebrick mint 31 Manufacturer#1 almond antique burnished rose metallic 2 Manufacturer#1 almond antique burnished rose metallic 2 -Manufacturer#3 almond antique chartreuse khaki white 17 Manufacturer#1 almond antique chartreuse lavender yellow 34 -Manufacturer#3 almond antique forest lavender goldenrod 14 -Manufacturer#4 almond antique gainsboro frosted violet 10 -Manufacturer#5 almond antique medium spring khaki 6 -Manufacturer#3 almond antique metallic orange dim 19 -Manufacturer#3 almond antique misty red olive 1 -Manufacturer#3 almond antique olive coral navajo 45 Manufacturer#1 almond antique salmon chartreuse burlywood 6 -Manufacturer#5 almond antique sky peru orange 2 +Manufacturer#1 almond aquamarine burnished black steel 28 +Manufacturer#1 almond aquamarine pink moccasin thistle 42 Manufacturer#2 almond antique violet chocolate turquoise 14 -Manufacturer#4 almond antique violet mint lemon 39 Manufacturer#2 almond antique violet turquoise frosted 40 -Manufacturer#1 almond aquamarine burnished black steel 28 -Manufacturer#5 almond aquamarine dodger light gainsboro 46 -Manufacturer#4 almond aquamarine floral ivory bisque 27 Manufacturer#2 almond aquamarine midnight light salmon 2 -Manufacturer#1 almond aquamarine pink moccasin thistle 42 Manufacturer#2 almond aquamarine rose maroon antique 25 Manufacturer#2 almond aquamarine sandy cyan gainsboro 18 +Manufacturer#3 almond antique chartreuse khaki white 17 +Manufacturer#3 almond antique forest lavender goldenrod 14 +Manufacturer#3 almond antique metallic orange dim 19 +Manufacturer#3 almond antique misty red olive 1 +Manufacturer#3 almond antique olive coral navajo 45 +Manufacturer#4 almond antique gainsboro frosted violet 10 +Manufacturer#4 almond antique violet mint lemon 39 +Manufacturer#4 almond aquamarine floral ivory bisque 27 Manufacturer#4 almond aquamarine yellow dodger mint 7 Manufacturer#4 almond azure aquamarine papaya violet 12 +Manufacturer#5 almond antique blue firebrick mint 31 +Manufacturer#5 almond antique medium spring khaki 6 +Manufacturer#5 almond antique sky peru orange 2 +Manufacturer#5 almond aquamarine dodger light gainsboro 46 Manufacturer#5 almond azure blanched chiffon midnight 23 PREHOOK: query: -- 2. testUDAFsNoWindowingNoPTFNoGBY select p_mfgr,p_name, p_retailprice, diff --git a/ql/src/test/results/clientpositive/spark/sample10.q.out b/ql/src/test/results/clientpositive/spark/sample10.q.out index a9574ee..33cfe84 100644 --- a/ql/src/test/results/clientpositive/spark/sample10.q.out +++ b/ql/src/test/results/clientpositive/spark/sample10.q.out @@ -89,7 +89,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 1) + Reducer 2 <- Map 1 (GROUP, 3) Reducer 3 <- Reducer 2 (SORT, 1) #### A masked pattern was here #### Vertices: diff --git a/ql/src/test/results/clientpositive/spark/semijoin.q.out b/ql/src/test/results/clientpositive/spark/semijoin.q.out index 9d527ce..d90d50d 100644 --- a/ql/src/test/results/clientpositive/spark/semijoin.q.out +++ b/ql/src/test/results/clientpositive/spark/semijoin.q.out @@ -158,7 +158,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 3 <- Map 2 (SORT, 1) + Reducer 3 <- Map 2 (SORT, 3) #### A masked pattern was here #### Vertices: Map 2 @@ -272,7 +272,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 3 <- Map 2 (SORT, 1) + Reducer 3 <- Map 2 (SORT, 3) #### A masked pattern was here #### Vertices: Map 2 @@ -388,7 +388,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 3 <- Map 2 (SORT, 1) + Reducer 3 <- Map 2 (SORT, 3) #### A masked pattern was here #### Vertices: Map 2 @@ -496,7 +496,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 3 <- Map 2 (SORT, 1) + Reducer 3 <- Map 2 (SORT, 3) #### A masked pattern was here #### Vertices: Map 2 @@ -615,7 +615,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 3 <- Map 2 (SORT, 1) + Reducer 3 <- Map 2 (SORT, 3) #### A masked pattern was here #### Vertices: Map 2 @@ -726,7 +726,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (SORT, 1) + Reducer 2 <- Map 1 (SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -837,7 +837,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 3 <- Map 2 (SORT, 1) + Reducer 3 <- Map 2 (SORT, 3) #### A masked pattern was here #### Vertices: Map 2 @@ -945,7 +945,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 3 <- Map 2 (SORT, 1) + Reducer 3 <- Map 2 (SORT, 3) #### A masked pattern was here #### Vertices: Map 2 @@ -1058,7 +1058,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 3 <- Map 2 (SORT, 1) + Reducer 3 <- Map 2 (SORT, 3) #### A masked pattern was here #### Vertices: Map 2 @@ -1185,7 +1185,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 3 <- Map 2 (SORT, 1) + Reducer 3 <- Map 2 (SORT, 3) #### A masked pattern was here #### Vertices: Map 2 @@ -1318,7 +1318,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (SORT, 1) + Reducer 2 <- Map 1 (SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -1447,7 +1447,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 3 <- Map 2 (SORT, 1) + Reducer 3 <- Map 2 (SORT, 3) #### A masked pattern was here #### Vertices: Map 2 @@ -1599,7 +1599,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 4 <- Map 3 (SORT, 1) + Reducer 4 <- Map 3 (SORT, 3) #### A masked pattern was here #### Vertices: Map 3 @@ -1742,7 +1742,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 4 <- Map 3 (SORT, 1) + Reducer 4 <- Map 3 (SORT, 3) #### A masked pattern was here #### Vertices: Map 3 @@ -1894,7 +1894,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (SORT, 1) + Reducer 2 <- Map 1 (SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -2002,8 +2002,8 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1), Map 5 (PARTITION-LEVEL SORT, 1) - Reducer 3 <- Reducer 2 (SORT, 1) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3), Map 5 (PARTITION-LEVEL SORT, 3) + Reducer 3 <- Reducer 2 (SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -2185,7 +2185,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 4 <- Map 3 (SORT, 1) + Reducer 4 <- Map 3 (SORT, 3) #### A masked pattern was here #### Vertices: Map 3 @@ -2340,7 +2340,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 3 <- Map 2 (SORT, 1) + Reducer 3 <- Map 2 (SORT, 3) #### A masked pattern was here #### Vertices: Map 2 @@ -2450,8 +2450,8 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1), Map 5 (PARTITION-LEVEL SORT, 1) - Reducer 3 <- Reducer 2 (SORT, 1) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3), Map 5 (PARTITION-LEVEL SORT, 3) + Reducer 3 <- Reducer 2 (SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -2645,7 +2645,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 4 <- Map 3 (SORT, 1) + Reducer 4 <- Map 3 (SORT, 3) #### A masked pattern was here #### Vertices: Map 3 diff --git a/ql/src/test/results/clientpositive/spark/skewjoinopt2.q.out b/ql/src/test/results/clientpositive/spark/skewjoinopt2.q.out index 4cecdf2..b0f0706 100644 --- a/ql/src/test/results/clientpositive/spark/skewjoinopt2.q.out +++ b/ql/src/test/results/clientpositive/spark/skewjoinopt2.q.out @@ -399,7 +399,7 @@ STAGE PLANS: Spark Edges: Union 2 <- Map 1 (NONE, 0), Map 6 (NONE, 0) - Reducer 3 <- Union 2 (GROUP, 1) + Reducer 3 <- Union 2 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -573,7 +573,7 @@ STAGE PLANS: Spark Edges: Union 2 <- Map 1 (NONE, 0), Map 6 (NONE, 0) - Reducer 3 <- Union 2 (GROUP, 1) + Reducer 3 <- Union 2 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 1 diff --git a/ql/src/test/results/clientpositive/spark/skewjoinopt3.q.out b/ql/src/test/results/clientpositive/spark/skewjoinopt3.q.out index 49405da..c35400a 100644 --- a/ql/src/test/results/clientpositive/spark/skewjoinopt3.q.out +++ b/ql/src/test/results/clientpositive/spark/skewjoinopt3.q.out @@ -214,8 +214,8 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1) - Reducer 6 <- Map 5 (PARTITION-LEVEL SORT, 1), Map 7 (PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3) + Reducer 6 <- Map 5 (PARTITION-LEVEL SORT, 3), Map 7 (PARTITION-LEVEL SORT, 3) Union 3 <- Reducer 2 (NONE, 0), Reducer 6 (NONE, 0) #### A masked pattern was here #### Vertices: diff --git a/ql/src/test/results/clientpositive/spark/skewjoinopt9.q.out b/ql/src/test/results/clientpositive/spark/skewjoinopt9.q.out index ecb8a65..1019c2e 100644 --- a/ql/src/test/results/clientpositive/spark/skewjoinopt9.q.out +++ b/ql/src/test/results/clientpositive/spark/skewjoinopt9.q.out @@ -225,7 +225,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 1) + Reducer 2 <- Map 1 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 1 diff --git a/ql/src/test/results/clientpositive/spark/smb_mapjoin_14.q.out b/ql/src/test/results/clientpositive/spark/smb_mapjoin_14.q.out index 4bbc455..fc4d7f0 100644 --- a/ql/src/test/results/clientpositive/spark/smb_mapjoin_14.q.out +++ b/ql/src/test/results/clientpositive/spark/smb_mapjoin_14.q.out @@ -208,7 +208,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 1) + Reducer 2 <- Map 1 (GROUP, 3) Reducer 3 <- Reducer 2 (SORT, 1) #### A masked pattern was here #### Vertices: @@ -367,7 +367,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 3 <- Map 2 (GROUP, 1) + Reducer 3 <- Map 2 (GROUP, 3) Reducer 4 <- Reducer 3 (GROUP, 1) #### A masked pattern was here #### Vertices: diff --git a/ql/src/test/results/clientpositive/spark/smb_mapjoin_17.q.out b/ql/src/test/results/clientpositive/spark/smb_mapjoin_17.q.out index 49b9fda..6c736af 100644 --- a/ql/src/test/results/clientpositive/spark/smb_mapjoin_17.q.out +++ b/ql/src/test/results/clientpositive/spark/smb_mapjoin_17.q.out @@ -781,7 +781,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 3 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1), Map 5 (PARTITION-LEVEL SORT, 1), Map 6 (PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 3 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3), Map 5 (PARTITION-LEVEL SORT, 3), Map 6 (PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 diff --git a/ql/src/test/results/clientpositive/spark/smb_mapjoin_7.q.out b/ql/src/test/results/clientpositive/spark/smb_mapjoin_7.q.out index 363207a..b040a6a 100644 --- a/ql/src/test/results/clientpositive/spark/smb_mapjoin_7.q.out +++ b/ql/src/test/results/clientpositive/spark/smb_mapjoin_7.q.out @@ -623,7 +623,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 3 (PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 3 (PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 diff --git a/ql/src/test/results/clientpositive/spark/subquery_in.q.out b/ql/src/test/results/clientpositive/spark/subquery_in.q.out index 4bb6290..552b5ae 100644 --- a/ql/src/test/results/clientpositive/spark/subquery_in.q.out +++ b/ql/src/test/results/clientpositive/spark/subquery_in.q.out @@ -281,7 +281,7 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3) Reducer 3 <- Reducer 2 (GROUP, 1) #### A masked pattern was here #### Vertices: @@ -441,8 +441,8 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 3 <- Map 2 (PARTITION-LEVEL SORT, 1) - Reducer 4 <- Reducer 3 (GROUP, 1) + Reducer 3 <- Map 2 (PARTITION-LEVEL SORT, 3) + Reducer 4 <- Reducer 3 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 2 @@ -609,7 +609,7 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 3 <- Map 2 (GROUP, 1) + Reducer 3 <- Map 2 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 2 @@ -829,7 +829,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 3 <- Map 2 (GROUP, 1) + Reducer 3 <- Map 2 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 2 diff --git a/ql/src/test/results/clientpositive/spark/table_access_keys_stats.q.out b/ql/src/test/results/clientpositive/spark/table_access_keys_stats.q.out index fbe392e..deaf5f2 100644 --- a/ql/src/test/results/clientpositive/spark/table_access_keys_stats.q.out +++ b/ql/src/test/results/clientpositive/spark/table_access_keys_stats.q.out @@ -209,7 +209,6 @@ Keys:key 2 1 2 12 1 3 1 3 13 1 7 1 7 17 1 -8 2 8 18 1 8 2 8 28 1 PREHOOK: query: -- constants from sub-queries should work fine SELECT key, constant, val, count(1) from diff --git a/ql/src/test/results/clientpositive/spark/union25.q.out b/ql/src/test/results/clientpositive/spark/union25.q.out index abb93b6..b64ec53 100644 --- a/ql/src/test/results/clientpositive/spark/union25.q.out +++ b/ql/src/test/results/clientpositive/spark/union25.q.out @@ -68,8 +68,8 @@ STAGE PLANS: Edges: Union 2 <- Map 1 (NONE, 0), Map 6 (NONE, 0) Union 4 <- Map 7 (NONE, 0), Reducer 3 (NONE, 0) - Reducer 3 <- Union 2 (GROUP, 1) - Reducer 5 <- Union 4 (GROUP, 1) + Reducer 3 <- Union 2 (GROUP, 3) + Reducer 5 <- Union 4 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 1 diff --git a/ql/src/test/results/clientpositive/spark/union33.q.out b/ql/src/test/results/clientpositive/spark/union33.q.out index 0a81d4a..1fbc5b5 100644 --- a/ql/src/test/results/clientpositive/spark/union33.q.out +++ b/ql/src/test/results/clientpositive/spark/union33.q.out @@ -196,9 +196,9 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 3) Union 4 <- Map 5 (NONE, 0), Reducer 3 (NONE, 0) - Reducer 3 <- Reducer 2 (GROUP, 1) + Reducer 3 <- Reducer 2 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 1 diff --git a/ql/src/test/results/clientpositive/spark/union_remove_19.q.out b/ql/src/test/results/clientpositive/spark/union_remove_19.q.out index 41daaf6..d6d4add 100644 --- a/ql/src/test/results/clientpositive/spark/union_remove_19.q.out +++ b/ql/src/test/results/clientpositive/spark/union_remove_19.q.out @@ -276,8 +276,8 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 1) - Reducer 5 <- Map 4 (GROUP, 1) + Reducer 2 <- Map 1 (GROUP, 3) + Reducer 5 <- Map 4 (GROUP, 3) Union 3 <- Reducer 2 (NONE, 0), Reducer 5 (NONE, 0) #### A masked pattern was here #### Vertices: @@ -448,8 +448,8 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 1) - Reducer 5 <- Map 4 (GROUP, 1) + Reducer 2 <- Map 1 (GROUP, 3) + Reducer 5 <- Map 4 (GROUP, 3) Union 3 <- Reducer 2 (NONE, 0), Reducer 5 (NONE, 0) #### A masked pattern was here #### Vertices: diff --git a/ql/src/test/results/clientpositive/spark/vector_cast_constant.q.out b/ql/src/test/results/clientpositive/spark/vector_cast_constant.q.out index 5ff382f..e43a0ca 100644 --- a/ql/src/test/results/clientpositive/spark/vector_cast_constant.q.out +++ b/ql/src/test/results/clientpositive/spark/vector_cast_constant.q.out @@ -119,7 +119,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 1) + Reducer 2 <- Map 1 (GROUP, 3) Reducer 3 <- Reducer 2 (SORT, 1) #### A masked pattern was here #### Vertices: diff --git a/ql/src/test/results/clientpositive/spark/vector_distinct_2.q.out b/ql/src/test/results/clientpositive/spark/vector_distinct_2.q.out index 7304a0c..279c61a 100644 --- a/ql/src/test/results/clientpositive/spark/vector_distinct_2.q.out +++ b/ql/src/test/results/clientpositive/spark/vector_distinct_2.q.out @@ -119,7 +119,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 1) + Reducer 2 <- Map 1 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 1 diff --git a/ql/src/test/results/clientpositive/spark/vector_groupby_3.q.out b/ql/src/test/results/clientpositive/spark/vector_groupby_3.q.out index 72cc00f..32bddd3 100644 --- a/ql/src/test/results/clientpositive/spark/vector_groupby_3.q.out +++ b/ql/src/test/results/clientpositive/spark/vector_groupby_3.q.out @@ -119,7 +119,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 1) + Reducer 2 <- Map 1 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 1 diff --git a/ql/src/test/results/clientpositive/spark/vector_mapjoin_reduce.q.out b/ql/src/test/results/clientpositive/spark/vector_mapjoin_reduce.q.out index b6a36d8..1927626 100644 --- a/ql/src/test/results/clientpositive/spark/vector_mapjoin_reduce.q.out +++ b/ql/src/test/results/clientpositive/spark/vector_mapjoin_reduce.q.out @@ -257,7 +257,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 3 <- Map 2 (GROUP, 1) + Reducer 3 <- Map 2 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 2 diff --git a/ql/src/test/results/clientpositive/spark/vector_orderby_5.q.out b/ql/src/test/results/clientpositive/spark/vector_orderby_5.q.out index 6a7c990..0992573 100644 --- a/ql/src/test/results/clientpositive/spark/vector_orderby_5.q.out +++ b/ql/src/test/results/clientpositive/spark/vector_orderby_5.q.out @@ -115,7 +115,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 1) + Reducer 2 <- Map 1 (GROUP, 3) Reducer 3 <- Reducer 2 (SORT, 1) #### A masked pattern was here #### Vertices: diff --git a/ql/src/test/results/clientpositive/spark/vectorization_short_regress.q.out b/ql/src/test/results/clientpositive/spark/vectorization_short_regress.q.out index d02c96e..269c4e3 100644 --- a/ql/src/test/results/clientpositive/spark/vectorization_short_regress.q.out +++ b/ql/src/test/results/clientpositive/spark/vectorization_short_regress.q.out @@ -1866,7 +1866,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 1) + Reducer 2 <- Map 1 (GROUP, 3) Reducer 3 <- Reducer 2 (SORT, 1) #### A masked pattern was here #### Vertices: @@ -2074,7 +2074,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 1) + Reducer 2 <- Map 1 (GROUP, 3) Reducer 3 <- Reducer 2 (SORT, 1) #### A masked pattern was here #### Vertices: @@ -2329,7 +2329,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 1) + Reducer 2 <- Map 1 (GROUP, 3) Reducer 3 <- Reducer 2 (SORT, 1) #### A masked pattern was here #### Vertices: @@ -2662,7 +2662,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 1) + Reducer 2 <- Map 1 (GROUP, 3) Reducer 3 <- Reducer 2 (SORT, 1) #### A masked pattern was here #### Vertices: diff --git a/ql/src/test/results/clientpositive/spark/vectorized_ptf.q.out b/ql/src/test/results/clientpositive/spark/vectorized_ptf.q.out index 1028345..5fe0020 100644 --- a/ql/src/test/results/clientpositive/spark/vectorized_ptf.q.out +++ b/ql/src/test/results/clientpositive/spark/vectorized_ptf.q.out @@ -228,8 +228,8 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1) - Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3) + Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -588,8 +588,8 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1) - Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3) + Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -839,7 +839,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -1095,8 +1095,8 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1) - Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3) + Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -1383,8 +1383,8 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1) - Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3) + Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -1680,9 +1680,9 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1) - Reducer 3 <- Reducer 2 (GROUP, 1) - Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3) + Reducer 3 <- Reducer 2 (GROUP, 3) + Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -2020,7 +2020,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 3 <- Map 2 (PARTITION-LEVEL SORT, 1) + Reducer 3 <- Map 2 (PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 2 @@ -2263,7 +2263,7 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 3 <- Map 2 (PARTITION-LEVEL SORT, 1) + Reducer 3 <- Map 2 (PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 2 @@ -2589,8 +2589,8 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1) - Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3) + Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -2866,8 +2866,8 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1) - Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3) + Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -3147,8 +3147,8 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1) - Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3) + Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -3435,9 +3435,9 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1) - Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 1) - Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3) + Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3) + Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -3762,8 +3762,8 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1) - Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3) + Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -4222,8 +4222,8 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 3 <- Map 2 (PARTITION-LEVEL SORT, 1) - Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 1) + Reducer 3 <- Map 2 (PARTITION-LEVEL SORT, 3) + Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 2 @@ -4478,8 +4478,8 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1) - Reducer 3 <- Reducer 2 (GROUP, 1) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3) + Reducer 3 <- Reducer 2 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -4749,9 +4749,9 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 1) - Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 1) - Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (GROUP, 3) + Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3) + Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -5222,11 +5222,11 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 6 <- Map 1 (PARTITION-LEVEL SORT, 1) - Reducer 7 <- Map 1 (PARTITION-LEVEL SORT, 1) - Reducer 3 <- Reducer 6 (PARTITION-LEVEL SORT, 1) - Reducer 5 <- Reducer 7 (PARTITION-LEVEL SORT, 1) - Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 1) + Reducer 6 <- Map 1 (PARTITION-LEVEL SORT, 3) + Reducer 7 <- Map 1 (PARTITION-LEVEL SORT, 3) + Reducer 3 <- Reducer 6 (PARTITION-LEVEL SORT, 3) + Reducer 5 <- Reducer 7 (PARTITION-LEVEL SORT, 3) + Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -5736,9 +5736,9 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1) - Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 1) - Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3) + Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3) + Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -6092,10 +6092,10 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1) - Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 1) - Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 1) - Reducer 5 <- Reducer 4 (PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3) + Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3) + Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 3) + Reducer 5 <- Reducer 4 (PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -6446,9 +6446,9 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1) - Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 1) - Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3) + Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3) + Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -6795,10 +6795,10 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1) - Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 1) - Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 1) - Reducer 5 <- Reducer 4 (PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3) + Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3) + Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 3) + Reducer 5 <- Reducer 4 (PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -7194,9 +7194,9 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1) - Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 1) - Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3) + Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3) + Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -7543,9 +7543,9 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1) - Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 1) - Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3) + Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3) + Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 diff --git a/ql/src/test/results/clientpositive/tez/count.q.out b/ql/src/test/results/clientpositive/tez/count.q.out index fe63101..026b5ff 100644 --- a/ql/src/test/results/clientpositive/tez/count.q.out +++ b/ql/src/test/results/clientpositive/tez/count.q.out @@ -1,8 +1,10 @@ -PREHOOK: query: create table abcd (a int, b int, c int, d int) +PREHOOK: query: -- SORT_QUERY_RESULTS +create table abcd (a int, b int, c int, d int) PREHOOK: type: CREATETABLE PREHOOK: Output: database:default PREHOOK: Output: default@abcd -POSTHOOK: query: create table abcd (a int, b int, c int, d int) +POSTHOOK: query: -- SORT_QUERY_RESULTS +create table abcd (a int, b int, c int, d int) POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@abcd @@ -22,13 +24,13 @@ POSTHOOK: query: select * from abcd POSTHOOK: type: QUERY POSTHOOK: Input: default@abcd #### A masked pattern was here #### -NULL 35 23 6 +10 100 45 4 +10 100 NULL 5 10 1000 50 1 100 100 10 3 -12 NULL 80 2 -10 100 NULL 5 -10 100 45 4 12 100 75 7 +12 NULL 80 2 +NULL 35 23 6 PREHOOK: query: explain select a, count(distinct b), count(distinct c), sum(d) from abcd group by a PREHOOK: type: QUERY POSTHOOK: query: explain select a, count(distinct b), count(distinct c), sum(d) from abcd group by a @@ -99,10 +101,10 @@ POSTHOOK: query: select a, count(distinct b), count(distinct c), sum(d) from abc POSTHOOK: type: QUERY POSTHOOK: Input: default@abcd #### A masked pattern was here #### -NULL 1 1 6 10 2 2 10 -12 1 2 9 100 1 1 3 +12 1 2 9 +NULL 1 1 6 PREHOOK: query: explain select count(1), count(*), count(a), count(b), count(c), count(d), count(distinct a), count(distinct b), count(distinct c), count(distinct d), count(distinct a,b), count(distinct b,c), count(distinct c,d), count(distinct a,d), count(distinct a,c), count(distinct b,d), count(distinct a,b,c), count(distinct b,c,d), count(distinct a,c,d), count(distinct a,b,d), count(distinct a,b,c,d) from abcd PREHOOK: type: QUERY POSTHOOK: query: explain select count(1), count(*), count(a), count(b), count(c), count(d), count(distinct a), count(distinct b), count(distinct c), count(distinct d), count(distinct a,b), count(distinct b,c), count(distinct c,d), count(distinct a,d), count(distinct a,c), count(distinct b,d), count(distinct a,b,c), count(distinct b,c,d), count(distinct a,c,d), count(distinct a,b,d), count(distinct a,b,c,d) from abcd @@ -236,10 +238,10 @@ POSTHOOK: query: select a, count(distinct b), count(distinct c), sum(d) from abc POSTHOOK: type: QUERY POSTHOOK: Input: default@abcd #### A masked pattern was here #### -NULL 1 1 6 10 2 2 10 -12 1 2 9 100 1 1 3 +12 1 2 9 +NULL 1 1 6 PREHOOK: query: explain select count(1), count(*), count(a), count(b), count(c), count(d), count(distinct a), count(distinct b), count(distinct c), count(distinct d), count(distinct a,b), count(distinct b,c), count(distinct c,d), count(distinct a,d), count(distinct a,c), count(distinct b,d), count(distinct a,b,c), count(distinct b,c,d), count(distinct a,c,d), count(distinct a,b,d), count(distinct a,b,c,d) from abcd PREHOOK: type: QUERY POSTHOOK: query: explain select count(1), count(*), count(a), count(b), count(c), count(d), count(distinct a), count(distinct b), count(distinct c), count(distinct d), count(distinct a,b), count(distinct b,c), count(distinct c,d), count(distinct a,d), count(distinct a,c), count(distinct b,d), count(distinct a,b,c), count(distinct b,c,d), count(distinct a,c,d), count(distinct a,b,d), count(distinct a,b,c,d) from abcd diff --git a/ql/src/test/results/clientpositive/tez/limit_pushdown.q.out b/ql/src/test/results/clientpositive/tez/limit_pushdown.q.out index c765ab7..d5d6e68 100644 --- a/ql/src/test/results/clientpositive/tez/limit_pushdown.q.out +++ b/ql/src/test/results/clientpositive/tez/limit_pushdown.q.out @@ -173,10 +173,10 @@ POSTHOOK: Input: default@src 98 val_98 98 val_98 PREHOOK: query: explain -select value, sum(key + 1) as sum from src group by value limit 20 +select value, sum(key + 1) as sum from src group by value order by value limit 20 PREHOOK: type: QUERY POSTHOOK: query: explain -select value, sum(key + 1) as sum from src group by value limit 20 +select value, sum(key + 1) as sum from src group by value order by value limit 20 POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage @@ -240,11 +240,11 @@ STAGE PLANS: Processor Tree: ListSink -PREHOOK: query: select value, sum(key + 1) as sum from src group by value limit 20 +PREHOOK: query: select value, sum(key + 1) as sum from src group by value order by value limit 20 PREHOOK: type: QUERY PREHOOK: Input: default@src #### A masked pattern was here #### -POSTHOOK: query: select value, sum(key + 1) as sum from src group by value limit 20 +POSTHOOK: query: select value, sum(key + 1) as sum from src group by value order by value limit 20 POSTHOOK: type: QUERY POSTHOOK: Input: default@src #### A masked pattern was here #### @@ -462,10 +462,10 @@ POSTHOOK: Input: default@alltypesorc -16379.0 NULL PREHOOK: query: explain -select ctinyint, count(distinct(cdouble)) from alltypesorc group by ctinyint limit 20 +select ctinyint, count(distinct(cdouble)) from alltypesorc group by ctinyint order by ctinyint limit 20 PREHOOK: type: QUERY POSTHOOK: query: explain -select ctinyint, count(distinct(cdouble)) from alltypesorc group by ctinyint limit 20 +select ctinyint, count(distinct(cdouble)) from alltypesorc group by ctinyint order by ctinyint limit 20 POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage @@ -528,11 +528,11 @@ STAGE PLANS: Processor Tree: ListSink -PREHOOK: query: select ctinyint, count(distinct(cdouble)) from alltypesorc group by ctinyint limit 20 +PREHOOK: query: select ctinyint, count(distinct(cdouble)) from alltypesorc group by ctinyint order by ctinyint limit 20 PREHOOK: type: QUERY PREHOOK: Input: default@alltypesorc #### A masked pattern was here #### -POSTHOOK: query: select ctinyint, count(distinct(cdouble)) from alltypesorc group by ctinyint limit 20 +POSTHOOK: query: select ctinyint, count(distinct(cdouble)) from alltypesorc group by ctinyint order by ctinyint limit 20 POSTHOOK: type: QUERY POSTHOOK: Input: default@alltypesorc #### A masked pattern was here #### @@ -558,11 +558,11 @@ POSTHOOK: Input: default@alltypesorc NULL 2932 PREHOOK: query: -- multi distinct explain -select ctinyint, count(distinct(cstring1)), count(distinct(cstring2)) from alltypesorc group by ctinyint limit 20 +select ctinyint, count(distinct(cstring1)), count(distinct(cstring2)) from alltypesorc group by ctinyint order by ctinyint limit 20 PREHOOK: type: QUERY POSTHOOK: query: -- multi distinct explain -select ctinyint, count(distinct(cstring1)), count(distinct(cstring2)) from alltypesorc group by ctinyint limit 20 +select ctinyint, count(distinct(cstring1)), count(distinct(cstring2)) from alltypesorc group by ctinyint order by ctinyint limit 20 POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage @@ -625,11 +625,11 @@ STAGE PLANS: Processor Tree: ListSink -PREHOOK: query: select ctinyint, count(distinct(cstring1)), count(distinct(cstring2)) from alltypesorc group by ctinyint limit 20 +PREHOOK: query: select ctinyint, count(distinct(cstring1)), count(distinct(cstring2)) from alltypesorc group by ctinyint order by ctinyint limit 20 PREHOOK: type: QUERY PREHOOK: Input: default@alltypesorc #### A masked pattern was here #### -POSTHOOK: query: select ctinyint, count(distinct(cstring1)), count(distinct(cstring2)) from alltypesorc group by ctinyint limit 20 +POSTHOOK: query: select ctinyint, count(distinct(cstring1)), count(distinct(cstring2)) from alltypesorc group by ctinyint order by ctinyint limit 20 POSTHOOK: type: QUERY POSTHOOK: Input: default@alltypesorc #### A masked pattern was here #### @@ -958,11 +958,11 @@ STAGE PLANS: PREHOOK: query: -- map aggregation disabled explain -select value, sum(key) as sum from src group by value limit 20 +select value, sum(key) as sum from src group by value order by value limit 20 PREHOOK: type: QUERY POSTHOOK: query: -- map aggregation disabled explain -select value, sum(key) as sum from src group by value limit 20 +select value, sum(key) as sum from src group by value order by value limit 20 POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage @@ -1020,11 +1020,11 @@ STAGE PLANS: Processor Tree: ListSink -PREHOOK: query: select value, sum(key) as sum from src group by value limit 20 +PREHOOK: query: select value, sum(key) as sum from src group by value order by value limit 20 PREHOOK: type: QUERY PREHOOK: Input: default@src #### A masked pattern was here #### -POSTHOOK: query: select value, sum(key) as sum from src group by value limit 20 +POSTHOOK: query: select value, sum(key) as sum from src group by value order by value limit 20 POSTHOOK: type: QUERY POSTHOOK: Input: default@src #### A masked pattern was here #### diff --git a/ql/src/test/results/clientpositive/tez/mapreduce2.q.out b/ql/src/test/results/clientpositive/tez/mapreduce2.q.out index 373afe3..06c56d6 100644 --- a/ql/src/test/results/clientpositive/tez/mapreduce2.q.out +++ b/ql/src/test/results/clientpositive/tez/mapreduce2.q.out @@ -105,11 +105,11 @@ POSTHOOK: Lineage: dest1.key SCRIPT [(src)src.FieldSchema(name:key, type:string, POSTHOOK: Lineage: dest1.one SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: dest1.ten SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: dest1.value SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ] -PREHOOK: query: SELECT * FROM (SELECT dest1.* FROM dest1 DISTRIBUTE BY key SORT BY key, ten, one, value) T +PREHOOK: query: SELECT * FROM (SELECT dest1.* FROM dest1 DISTRIBUTE BY key SORT BY key, ten, one, value) T ORDER BY key PREHOOK: type: QUERY PREHOOK: Input: default@dest1 #### A masked pattern was here #### -POSTHOOK: query: SELECT * FROM (SELECT dest1.* FROM dest1 DISTRIBUTE BY key SORT BY key, ten, one, value) T +POSTHOOK: query: SELECT * FROM (SELECT dest1.* FROM dest1 DISTRIBUTE BY key SORT BY key, ten, one, value) T ORDER BY key POSTHOOK: type: QUERY POSTHOOK: Input: default@dest1 #### A masked pattern was here #### diff --git a/spark-client/src/main/java/org/apache/hive/spark/client/ClientUtils.java b/spark-client/src/main/java/org/apache/hive/spark/client/ClientUtils.java index 47c13d1..98f5ad2 100644 --- a/spark-client/src/main/java/org/apache/hive/spark/client/ClientUtils.java +++ b/spark-client/src/main/java/org/apache/hive/spark/client/ClientUtils.java @@ -64,7 +64,7 @@ static ActorSystemInfo createActorSystem(Map conf) throws IOExce int akkaHeartBeatPauses = toInt(conf.get("spark.akka.heartbeat.pauses"), 600); double akkaFailureDetector = toDouble(conf.get("spark.akka.failure-detector.threshold"), 300.0); - int akkaHeartBeatInterval = toInt(conf.get("spark.akka.heartbeat.interval"), 1000); + int akkaHeartBeatInterval = toInt(conf.get("spark.akka.heartbeat.interval"), 100); // Disabled due to chill-akka depending on kryo 2.21, which is incompatible with 2.22 // due to packaging changes (relocated org.objenesis classes). diff --git a/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientImpl.java b/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientImpl.java index 0aea4a2..903169d 100644 --- a/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientImpl.java +++ b/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientImpl.java @@ -17,10 +17,12 @@ package org.apache.hive.spark.client; +import java.io.BufferedReader; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; +import java.io.InputStreamReader; import java.io.OutputStream; import java.io.OutputStreamWriter; import java.io.Serializable; @@ -192,8 +194,12 @@ public void run() { // If a Spark installation is provided, use the spark-submit script. Otherwise, call the // SparkSubmit class directly, which has some caveats (like having to provide a proper // version of Guava on the classpath depending on the deploy mode). - if (conf.get("spark.home") != null) { - argv.add(new File(conf.get("spark.home"), "bin/spark-submit").getAbsolutePath()); + String sparkHome = conf.get("spark.home"); + if (sparkHome == null) { + sparkHome = System.getProperty("spark.home"); + } + if (sparkHome != null) { + argv.add(new File(sparkHome, "bin/spark-submit").getAbsolutePath()); } else { LOG.info("No spark.home provided, calling SparkSubmit directly."); argv.add(new File(System.getProperty("java.home"), "bin/java").getAbsolutePath()); @@ -248,12 +254,11 @@ public void run() { LOG.debug("Running client driver with argv: {}", Joiner.on(" ").join(argv)); ProcessBuilder pb = new ProcessBuilder(argv.toArray(new String[argv.size()])); - pb.environment().clear(); final Process child = pb.start(); int childId = childIdGenerator.incrementAndGet(); - redirect("stdout-redir-" + childId, child.getInputStream(), System.out); - redirect("stderr-redir-" + childId, child.getErrorStream(), System.err); + redirect("stdout-redir-" + childId, child.getInputStream()); + redirect("stderr-redir-" + childId, child.getErrorStream()); runnable = new Runnable() { @Override @@ -277,8 +282,8 @@ public void run() { return thread; } - private void redirect(String name, InputStream in, OutputStream out) { - Thread thread = new Thread(new Redirector(in, out)); + private void redirect(String name, InputStream in) { + Thread thread = new Thread(new Redirector(in)); thread.setName(name); thread.setDaemon(true); thread.start(); @@ -345,23 +350,18 @@ public void onReceive(Object message) throws Exception { private class Redirector implements Runnable { - private final InputStream in; - private final OutputStream out; + private final BufferedReader in; - Redirector(InputStream in, OutputStream out) { - this.in = in; - this.out = out; + Redirector(InputStream in) { + this.in = new BufferedReader(new InputStreamReader(in)); } @Override public void run() { try { - byte[] buf = new byte[1024]; - int len = in.read(buf); - while (len != -1) { - out.write(buf, 0, len); - out.flush(); - len = in.read(buf); + String line = null; + while ((line = in.readLine()) != null) { + LOG.info(line); } } catch (Exception e) { LOG.warn("Error in redirector thread.", e);