diff --git a/data/conf/hive-log4j.properties b/data/conf/hive-log4j.properties index f90b833..922efb8 100644 --- a/data/conf/hive-log4j.properties +++ b/data/conf/hive-log4j.properties @@ -42,7 +42,7 @@ log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout # Pattern format: Date LogLevel LoggerName LogMessage #log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n # Debugging Pattern format -log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n +log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p [%t]: %c{2} (%F:%M(%L)) - %m%n # diff --git a/data/conf/spark/hive-site.xml b/data/conf/spark/hive-site.xml index f3d7b40..e42aa22 100644 --- a/data/conf/spark/hive-site.xml +++ b/data/conf/spark/hive-site.xml @@ -197,7 +197,17 @@ spark.master - local[4] + local-cluster[2,2,1024] + + + + spark.akka.logLifecycleEvents + true + + + + spark.root.dir + ${spark.home} diff --git a/data/conf/spark/log4j.properties b/data/conf/spark/log4j.properties new file mode 100644 index 0000000..e884bdf --- /dev/null +++ b/data/conf/spark/log4j.properties @@ -0,0 +1,30 @@ +# Set everything to be logged to the console +log4j.rootCategory=INFO, DRFA +log4j.appender.console=org.apache.log4j.ConsoleAppender +log4j.appender.console.target=System.err +log4j.appender.console.layout=org.apache.log4j.PatternLayout +log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{1}: %m%n + +# Settings to quiet third party logs that are too verbose +log4j.logger.org.eclipse.jetty=WARN +log4j.logger.org.eclipse.jetty.util.component.AbstractLifeCycle=ERROR +log4j.logger.org.apache.spark.repl.SparkIMain$exprTyper=INFO +log4j.logger.org.apache.spark.repl.SparkILoop$SparkILoopInterpreter=INFO + + + +log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender + +log4j.appender.DRFA.File=${spark.root.dir}/logs/spark.log + +# Rollver at midnight +log4j.appender.DRFA.DatePattern=.yyyy-MM-dd + +# 30-day backup +#log4j.appender.DRFA.MaxBackupIndex=30 +log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout + +# Pattern format: Date LogLevel LoggerName LogMessage +#log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n +# Debugging Pattern format +log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p [%t]: %c{2} (%F:%M(%L)) - %m%n diff --git a/itests/qtest-spark/.gitignore b/itests/qtest-spark/.gitignore new file mode 100644 index 0000000..c2ed135 --- /dev/null +++ b/itests/qtest-spark/.gitignore @@ -0,0 +1 @@ +thirdparty/ diff --git a/itests/qtest-spark/pom.xml b/itests/qtest-spark/pom.xml index a533e8d..1b02be5 100644 --- a/itests/qtest-spark/pom.xml +++ b/itests/qtest-spark/pom.xml @@ -339,6 +339,44 @@ maven-antrun-plugin + download-spark + generate-sources + + run + + + + + env + set -e + set -x + /bin/pwd + BASE_DIR=./target + HIVE_ROOT=$BASE_DIR/../../../ + DOWNLOAD_DIR=./thirdparty + download() { + url=$1; + finalName=$2 + tarName=$(basename $url) + rm -rf $BASE_DIR/$finalName + if [[ ! -f $DOWNLOAD_DIR/$tarName ]] + then + curl -Sso $DOWNLOAD_DIR/$tarName $url + fi + tar -zxf $DOWNLOAD_DIR/$tarName -C $BASE_DIR + mv $BASE_DIR/${finalName}* $BASE_DIR/$finalName + } + mkdir -p $DOWNLOAD_DIR + download "http://ec2-50-18-79-139.us-west-1.compute.amazonaws.com/data/spark-1.2.1-SNAPSHOT-bin-hadoop2-without-hive.tgz" "spark" + cp -f $HIVE_ROOT/data/conf/spark/log4j.properties $BASE_DIR/spark/conf/ + + + + + + + + generate-tests-sources generate-test-sources diff --git a/pom.xml b/pom.xml index cce2e5e..900dddb 100644 --- a/pom.xml +++ b/pom.xml @@ -171,6 +171,7 @@ 2.4.0 2.6.0 3.0.0 + ${basedir}/${hive.path.to.root}/itests/qtest-spark/target/spark @@ -836,6 +837,7 @@ true ${test.tmp.dir} + ${spark.home} ${test.tmp.dir} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HiveSparkClientFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HiveSparkClientFactory.java index 1898560..2ba75e0 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HiveSparkClientFactory.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HiveSparkClientFactory.java @@ -100,7 +100,7 @@ public static HiveSparkClient createHiveSparkClient(Configuration configuration) for (Map.Entry entry : hiveConf) { String propertyName = entry.getKey(); if (propertyName.startsWith("spark")) { - String value = entry.getValue(); + String value = hiveConf.get(propertyName); sparkConf.put(propertyName, value); LOG.info(String.format( "load spark configuration from hive configuration (%s -> %s).", diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/SparkJobMonitor.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/SparkJobMonitor.java index 72108cd..c075c35 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/SparkJobMonitor.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/SparkJobMonitor.java @@ -67,6 +67,9 @@ public int startMonitor() { while (true) { try { JobExecutionStatus state = sparkJobStatus.getState(); + if (LOG.isDebugEnabled()) { + console.printInfo("state = " + state); + } if (state != null && state != JobExecutionStatus.UNKNOWN && (state != lastState || state == JobExecutionStatus.RUNNING)) { lastState = state; diff --git a/ql/src/test/results/clientpositive/spark/auto_join20.q.out b/ql/src/test/results/clientpositive/spark/auto_join20.q.out index e089234..31ee1bc 100644 --- a/ql/src/test/results/clientpositive/spark/auto_join20.q.out +++ b/ql/src/test/results/clientpositive/spark/auto_join20.q.out @@ -249,7 +249,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (SORT, 1) + Reducer 2 <- Map 1 (SORT, 3) Reducer 3 <- Reducer 2 (GROUP, 1) #### A masked pattern was here #### Vertices: diff --git a/ql/src/test/results/clientpositive/spark/auto_join29.q.out b/ql/src/test/results/clientpositive/spark/auto_join29.q.out index 6702c1c..e618ce0 100644 --- a/ql/src/test/results/clientpositive/spark/auto_join29.q.out +++ b/ql/src/test/results/clientpositive/spark/auto_join29.q.out @@ -694,7 +694,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 4 <- Map 3 (SORT, 1) + Reducer 4 <- Map 3 (SORT, 3) #### A masked pattern was here #### Vertices: Map 3 @@ -1327,7 +1327,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (SORT, 1) + Reducer 2 <- Map 1 (SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -1969,7 +1969,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 3 <- Map 2 (SORT, 1) + Reducer 3 <- Map 2 (SORT, 3) #### A masked pattern was here #### Vertices: Map 2 @@ -2606,7 +2606,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 4 <- Map 3 (SORT, 1) + Reducer 4 <- Map 3 (SORT, 3) #### A masked pattern was here #### Vertices: Map 3 @@ -2738,7 +2738,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 3 <- Map 2 (SORT, 1) + Reducer 3 <- Map 2 (SORT, 3) #### A masked pattern was here #### Vertices: Map 2 @@ -3371,7 +3371,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 4 <- Map 3 (SORT, 1) + Reducer 4 <- Map 3 (SORT, 3) #### A masked pattern was here #### Vertices: Map 3 @@ -3504,7 +3504,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (SORT, 1) + Reducer 2 <- Map 1 (SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -3654,7 +3654,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 4 <- Map 3 (SORT, 1) + Reducer 4 <- Map 3 (SORT, 3) #### A masked pattern was here #### Vertices: Map 3 diff --git a/ql/src/test/results/clientpositive/spark/auto_join30.q.out b/ql/src/test/results/clientpositive/spark/auto_join30.q.out index f6dd445..b401af5 100644 --- a/ql/src/test/results/clientpositive/spark/auto_join30.q.out +++ b/ql/src/test/results/clientpositive/spark/auto_join30.q.out @@ -185,7 +185,7 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 5 <- Map 4 (SORT, 1) + Reducer 5 <- Map 4 (SORT, 3) #### A masked pattern was here #### Vertices: Map 4 @@ -221,7 +221,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (SORT, 1) + Reducer 2 <- Map 1 (SORT, 3) Reducer 3 <- Reducer 2 (GROUP, 1) #### A masked pattern was here #### Vertices: @@ -341,7 +341,7 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 2 <- Map 1 (SORT, 1) + Reducer 2 <- Map 1 (SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -376,7 +376,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 4 <- Map 3 (SORT, 1) + Reducer 4 <- Map 3 (SORT, 3) Reducer 5 <- Reducer 4 (GROUP, 1) #### A masked pattern was here #### Vertices: @@ -503,8 +503,8 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 2 <- Map 1 (SORT, 1) - Reducer 7 <- Map 6 (SORT, 1) + Reducer 2 <- Map 1 (SORT, 3) + Reducer 7 <- Map 6 (SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -579,7 +579,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 4 <- Map 3 (SORT, 1) + Reducer 4 <- Map 3 (SORT, 3) Reducer 5 <- Reducer 4 (GROUP, 1) #### A masked pattern was here #### Vertices: @@ -718,8 +718,8 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 2 <- Map 1 (SORT, 1) - Reducer 7 <- Map 6 (SORT, 1) + Reducer 2 <- Map 1 (SORT, 3) + Reducer 7 <- Map 6 (SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -788,7 +788,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 4 <- Map 3 (SORT, 1) + Reducer 4 <- Map 3 (SORT, 3) Reducer 5 <- Reducer 4 (GROUP, 1) #### A masked pattern was here #### Vertices: @@ -924,8 +924,8 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 2 <- Map 1 (SORT, 1) - Reducer 7 <- Map 6 (SORT, 1) + Reducer 2 <- Map 1 (SORT, 3) + Reducer 7 <- Map 6 (SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -994,7 +994,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 4 <- Map 3 (SORT, 1) + Reducer 4 <- Map 3 (SORT, 3) Reducer 5 <- Reducer 4 (GROUP, 1) #### A masked pattern was here #### Vertices: @@ -1130,8 +1130,8 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 5 <- Map 4 (SORT, 1) - Reducer 7 <- Map 6 (SORT, 1) + Reducer 5 <- Map 4 (SORT, 3) + Reducer 7 <- Map 6 (SORT, 3) #### A masked pattern was here #### Vertices: Map 4 @@ -1199,7 +1199,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (SORT, 1) + Reducer 2 <- Map 1 (SORT, 3) Reducer 3 <- Reducer 2 (GROUP, 1) #### A masked pattern was here #### Vertices: @@ -1336,8 +1336,8 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 5 <- Map 4 (SORT, 1) - Reducer 7 <- Map 6 (SORT, 1) + Reducer 5 <- Map 4 (SORT, 3) + Reducer 7 <- Map 6 (SORT, 3) #### A masked pattern was here #### Vertices: Map 4 @@ -1405,7 +1405,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (SORT, 1) + Reducer 2 <- Map 1 (SORT, 3) Reducer 3 <- Reducer 2 (GROUP, 1) #### A masked pattern was here #### Vertices: diff --git a/ql/src/test/results/clientpositive/spark/auto_join32.q.out b/ql/src/test/results/clientpositive/spark/auto_join32.q.out index 9a7a982..fff4d54 100644 --- a/ql/src/test/results/clientpositive/spark/auto_join32.q.out +++ b/ql/src/test/results/clientpositive/spark/auto_join32.q.out @@ -176,7 +176,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -298,7 +298,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -444,7 +444,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 diff --git a/ql/src/test/results/clientpositive/spark/auto_join_without_localtask.q.out b/ql/src/test/results/clientpositive/spark/auto_join_without_localtask.q.out index f5e8f88..26d2165 100644 --- a/ql/src/test/results/clientpositive/spark/auto_join_without_localtask.q.out +++ b/ql/src/test/results/clientpositive/spark/auto_join_without_localtask.q.out @@ -1,8 +1,8 @@ PREHOOK: query: explain -select a.* from src a join src b on a.key=b.key limit 40 +select a.* from src a join src b on a.key=b.key order by key, value limit 40 PREHOOK: type: QUERY POSTHOOK: query: explain -select a.* from src a join src b on a.key=b.key limit 40 +select a.* from src a join src b on a.key=b.key order by key, value limit 40 POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage @@ -12,7 +12,8 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 3 (PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1) + Reducer 3 <- Reducer 2 (SORT, 1) #### A masked pattern was here #### Vertices: Map 1 @@ -28,7 +29,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: key (type: string) Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - Map 3 + Map 4 Map Operator Tree: TableScan alias: a @@ -56,16 +57,26 @@ STAGE PLANS: expressions: _col0 (type: string), _col1 (type: string) outputColumnNames: _col0, _col1 Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE - Limit - Number of rows: 40 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + Reducer 3 + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 40 + Statistics: Num rows: 40 Data size: 400 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false Statistics: Num rows: 40 Data size: 400 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 40 Data size: 400 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -73,11 +84,11 @@ STAGE PLANS: Processor Tree: ListSink -PREHOOK: query: select a.* from src a join src b on a.key=b.key limit 40 +PREHOOK: query: select a.* from src a join src b on a.key=b.key order by key, value limit 40 PREHOOK: type: QUERY PREHOOK: Input: default@src #### A masked pattern was here #### -POSTHOOK: query: select a.* from src a join src b on a.key=b.key limit 40 +POSTHOOK: query: select a.* from src a join src b on a.key=b.key order by key, value limit 40 POSTHOOK: type: QUERY POSTHOOK: Input: default@src #### A masked pattern was here #### @@ -123,10 +134,10 @@ RUN: Stage-1:MAPRED 119 val_119 119 val_119 PREHOOK: query: explain -select a.* from src a join src b on a.key=b.key join src c on a.value=c.value limit 40 +select a.* from src a join src b on a.key=b.key join src c on a.value=c.value order by a.key, a.value limit 40 PREHOOK: type: QUERY POSTHOOK: query: explain -select a.* from src a join src b on a.key=b.key join src c on a.value=c.value limit 40 +select a.* from src a join src b on a.key=b.key join src c on a.value=c.value order by a.key, a.value limit 40 POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage @@ -136,8 +147,9 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 5 (PARTITION-LEVEL SORT, 1) - Reducer 3 <- Map 4 (PARTITION-LEVEL SORT, 1), Reducer 2 (PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 6 (PARTITION-LEVEL SORT, 3) + Reducer 3 <- Map 5 (PARTITION-LEVEL SORT, 3), Reducer 2 (PARTITION-LEVEL SORT, 3) + Reducer 4 <- Reducer 3 (SORT, 1) #### A masked pattern was here #### Vertices: Map 1 @@ -153,7 +165,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: key (type: string) Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - Map 4 + Map 5 Map Operator Tree: TableScan alias: c @@ -166,7 +178,7 @@ STAGE PLANS: sort order: + Map-reduce partition columns: value (type: string) Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE - Map 5 + Map 6 Map Operator Tree: TableScan alias: a @@ -210,16 +222,26 @@ STAGE PLANS: expressions: _col0 (type: string), _col1 (type: string) outputColumnNames: _col0, _col1 Statistics: Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE - Limit - Number of rows: 40 + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + sort order: ++ + Statistics: Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE + Reducer 4 + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 40 + Statistics: Num rows: 40 Data size: 400 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false Statistics: Num rows: 40 Data size: 400 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 40 Data size: 400 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator @@ -227,11 +249,11 @@ STAGE PLANS: Processor Tree: ListSink -PREHOOK: query: select a.* from src a join src b on a.key=b.key join src c on a.value=c.value limit 40 +PREHOOK: query: select a.* from src a join src b on a.key=b.key join src c on a.value=c.value order by a.key, a.value limit 40 PREHOOK: type: QUERY PREHOOK: Input: default@src #### A masked pattern was here #### -POSTHOOK: query: select a.* from src a join src b on a.key=b.key join src c on a.value=c.value limit 40 +POSTHOOK: query: select a.* from src a join src b on a.key=b.key join src c on a.value=c.value order by a.key, a.value limit 40 POSTHOOK: type: QUERY POSTHOOK: Input: default@src #### A masked pattern was here #### @@ -290,8 +312,8 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 5 (PARTITION-LEVEL SORT, 1) - Reducer 3 <- Map 4 (PARTITION-LEVEL SORT, 1), Reducer 2 (PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 5 (PARTITION-LEVEL SORT, 3) + Reducer 3 <- Map 4 (PARTITION-LEVEL SORT, 3), Reducer 2 (PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -390,14 +412,6 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@src #### A masked pattern was here #### RUN: Stage-1:MAPRED -103 val_103 -103 val_103 -103 val_103 -103 val_103 -103 val_103 -103 val_103 -103 val_103 -103 val_103 104 val_104 104 val_104 104 val_104 @@ -406,8 +420,6 @@ RUN: Stage-1:MAPRED 104 val_104 104 val_104 104 val_104 -105 val_105 -111 val_111 113 val_113 113 val_113 113 val_113 @@ -416,16 +428,26 @@ RUN: Stage-1:MAPRED 113 val_113 113 val_113 113 val_113 -114 val_114 116 val_116 -118 val_118 -118 val_118 -118 val_118 -118 val_118 -118 val_118 -118 val_118 -118 val_118 -118 val_118 +119 val_119 +119 val_119 +119 val_119 +119 val_119 +119 val_119 +119 val_119 +119 val_119 +119 val_119 +119 val_119 +119 val_119 +119 val_119 +119 val_119 +119 val_119 +119 val_119 +119 val_119 +119 val_119 +119 val_119 +119 val_119 +119 val_119 119 val_119 119 val_119 119 val_119 @@ -441,14 +463,6 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@src #### A masked pattern was here #### RUN: Stage-1:MAPRED -103 val_103 -103 val_103 -103 val_103 -103 val_103 -103 val_103 -103 val_103 -103 val_103 -103 val_103 104 val_104 104 val_104 104 val_104 @@ -457,8 +471,6 @@ RUN: Stage-1:MAPRED 104 val_104 104 val_104 104 val_104 -105 val_105 -111 val_111 113 val_113 113 val_113 113 val_113 @@ -467,16 +479,26 @@ RUN: Stage-1:MAPRED 113 val_113 113 val_113 113 val_113 -114 val_114 116 val_116 -118 val_118 -118 val_118 -118 val_118 -118 val_118 -118 val_118 -118 val_118 -118 val_118 -118 val_118 +119 val_119 +119 val_119 +119 val_119 +119 val_119 +119 val_119 +119 val_119 +119 val_119 +119 val_119 +119 val_119 +119 val_119 +119 val_119 +119 val_119 +119 val_119 +119 val_119 +119 val_119 +119 val_119 +119 val_119 +119 val_119 +119 val_119 119 val_119 119 val_119 119 val_119 diff --git a/ql/src/test/results/clientpositive/spark/auto_smb_mapjoin_14.q.out b/ql/src/test/results/clientpositive/spark/auto_smb_mapjoin_14.q.out index 390af23..c66b49d 100644 --- a/ql/src/test/results/clientpositive/spark/auto_smb_mapjoin_14.q.out +++ b/ql/src/test/results/clientpositive/spark/auto_smb_mapjoin_14.q.out @@ -157,7 +157,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 1) + Reducer 2 <- Map 1 (GROUP, 3) Reducer 3 <- Reducer 2 (GROUP, 1) #### A masked pattern was here #### Vertices: @@ -300,9 +300,9 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 1) - Reducer 5 <- Map 4 (GROUP, 1) - Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 1), Reducer 5 (PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (GROUP, 3) + Reducer 5 <- Map 4 (GROUP, 3) + Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3), Reducer 5 (PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -1639,7 +1639,7 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 1) + Reducer 2 <- Map 1 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 1 diff --git a/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_10.q.out b/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_10.q.out index 445958a..382c340 100644 --- a/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_10.q.out +++ b/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_10.q.out @@ -72,7 +72,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 3 <- Map 6 (PARTITION-LEVEL SORT, 1), Union 2 (PARTITION-LEVEL SORT, 1) + Reducer 3 <- Map 6 (PARTITION-LEVEL SORT, 3), Union 2 (PARTITION-LEVEL SORT, 3) Union 2 <- Map 1 (NONE, 0), Map 5 (NONE, 0) Reducer 4 <- Reducer 3 (GROUP, 1) #### A masked pattern was here #### @@ -227,7 +227,7 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 1) + Reducer 2 <- Map 1 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 1 diff --git a/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_6.q.out b/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_6.q.out index 4562430..731dd1a 100644 --- a/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_6.q.out +++ b/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_6.q.out @@ -96,7 +96,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3) Reducer 3 <- Reducer 2 (GROUP, 1) #### A masked pattern was here #### Vertices: @@ -204,7 +204,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3) Reducer 3 <- Reducer 2 (GROUP, 1) #### A masked pattern was here #### Vertices: @@ -312,7 +312,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3) Reducer 3 <- Reducer 2 (GROUP, 1) #### A masked pattern was here #### Vertices: @@ -416,7 +416,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1), Map 5 (PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3), Map 5 (PARTITION-LEVEL SORT, 3) Reducer 3 <- Reducer 2 (GROUP, 1) #### A masked pattern was here #### Vertices: @@ -613,7 +613,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3) Reducer 3 <- Reducer 2 (GROUP, 1) #### A masked pattern was here #### Vertices: @@ -717,7 +717,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3) Reducer 3 <- Reducer 2 (GROUP, 1) #### A masked pattern was here #### Vertices: @@ -821,7 +821,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1), Map 5 (PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3), Map 5 (PARTITION-LEVEL SORT, 3) Reducer 3 <- Reducer 2 (GROUP, 1) #### A masked pattern was here #### Vertices: @@ -1018,7 +1018,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3) Reducer 3 <- Reducer 2 (GROUP, 1) #### A masked pattern was here #### Vertices: diff --git a/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_9.q.out b/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_9.q.out index d69ab4a..4853aee 100644 --- a/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_9.q.out +++ b/ql/src/test/results/clientpositive/spark/auto_sortmerge_join_9.q.out @@ -151,7 +151,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 1) + Reducer 2 <- Map 1 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -262,7 +262,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 1) + Reducer 2 <- Map 1 (GROUP, 3) Reducer 3 <- Reducer 2 (GROUP, 1) #### A masked pattern was here #### Vertices: @@ -405,9 +405,9 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 1) - Reducer 5 <- Map 4 (GROUP, 1) - Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 1), Reducer 5 (PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (GROUP, 3) + Reducer 5 <- Map 4 (GROUP, 3) + Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3), Reducer 5 (PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -1700,7 +1700,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 1) + Reducer 2 <- Map 1 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -1811,7 +1811,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 1) + Reducer 2 <- Map 1 (GROUP, 3) Reducer 3 <- Reducer 2 (GROUP, 1) #### A masked pattern was here #### Vertices: @@ -1954,9 +1954,9 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 1) - Reducer 5 <- Map 4 (GROUP, 1) - Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 1), Reducer 5 (PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (GROUP, 3) + Reducer 5 <- Map 4 (GROUP, 3) + Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3), Reducer 5 (PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 diff --git a/ql/src/test/results/clientpositive/spark/bucket_map_join_tez1.q.out b/ql/src/test/results/clientpositive/spark/bucket_map_join_tez1.q.out index 2790a18..b0a81e8 100644 --- a/ql/src/test/results/clientpositive/spark/bucket_map_join_tez1.q.out +++ b/ql/src/test/results/clientpositive/spark/bucket_map_join_tez1.q.out @@ -212,7 +212,7 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 3 <- Map 2 (GROUP, 1) + Reducer 3 <- Map 2 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 2 @@ -348,7 +348,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 3 <- Map 2 (GROUP, 1) + Reducer 3 <- Map 2 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 2 @@ -499,7 +499,7 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 3 <- Map 2 (GROUP, 1) + Reducer 3 <- Map 2 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 2 @@ -852,7 +852,7 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 3 <- Map 2 (GROUP, 1) + Reducer 3 <- Map 2 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 2 @@ -965,7 +965,7 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 3 <- Map 2 (GROUP, 1) + Reducer 3 <- Map 2 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 2 diff --git a/ql/src/test/results/clientpositive/spark/bucket_map_join_tez2.q.out b/ql/src/test/results/clientpositive/spark/bucket_map_join_tez2.q.out index b6d05df..6d9cc20 100644 --- a/ql/src/test/results/clientpositive/spark/bucket_map_join_tez2.q.out +++ b/ql/src/test/results/clientpositive/spark/bucket_map_join_tez2.q.out @@ -592,7 +592,7 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 3 <- Map 2 (GROUP, 1) + Reducer 3 <- Map 2 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 2 @@ -696,7 +696,7 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 3 <- Map 2 (GROUP, 1) + Reducer 3 <- Map 2 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 2 diff --git a/ql/src/test/results/clientpositive/spark/count.q.out b/ql/src/test/results/clientpositive/spark/count.q.out index 89d5544..3e4ac5c 100644 --- a/ql/src/test/results/clientpositive/spark/count.q.out +++ b/ql/src/test/results/clientpositive/spark/count.q.out @@ -1,8 +1,10 @@ -PREHOOK: query: create table abcd (a int, b int, c int, d int) +PREHOOK: query: -- SORT_QUERY_RESULTS +create table abcd (a int, b int, c int, d int) PREHOOK: type: CREATETABLE PREHOOK: Output: database:default PREHOOK: Output: default@abcd -POSTHOOK: query: create table abcd (a int, b int, c int, d int) +POSTHOOK: query: -- SORT_QUERY_RESULTS +create table abcd (a int, b int, c int, d int) POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@abcd @@ -22,13 +24,13 @@ POSTHOOK: query: select * from abcd POSTHOOK: type: QUERY POSTHOOK: Input: default@abcd #### A masked pattern was here #### -NULL 35 23 6 +10 100 45 4 +10 100 NULL 5 10 1000 50 1 100 100 10 3 -12 NULL 80 2 -10 100 NULL 5 -10 100 45 4 12 100 75 7 +12 NULL 80 2 +NULL 35 23 6 PREHOOK: query: explain select a, count(distinct b), count(distinct c), sum(d) from abcd group by a PREHOOK: type: QUERY POSTHOOK: query: explain select a, count(distinct b), count(distinct c), sum(d) from abcd group by a @@ -99,10 +101,10 @@ POSTHOOK: query: select a, count(distinct b), count(distinct c), sum(d) from abc POSTHOOK: type: QUERY POSTHOOK: Input: default@abcd #### A masked pattern was here #### -NULL 1 1 6 10 2 2 10 -12 1 2 9 100 1 1 3 +12 1 2 9 +NULL 1 1 6 PREHOOK: query: explain select count(1), count(*), count(a), count(b), count(c), count(d), count(distinct a), count(distinct b), count(distinct c), count(distinct d), count(distinct a,b), count(distinct b,c), count(distinct c,d), count(distinct a,d), count(distinct a,c), count(distinct b,d), count(distinct a,b,c), count(distinct b,c,d), count(distinct a,c,d), count(distinct a,b,d), count(distinct a,b,c,d) from abcd PREHOOK: type: QUERY POSTHOOK: query: explain select count(1), count(*), count(a), count(b), count(c), count(d), count(distinct a), count(distinct b), count(distinct c), count(distinct d), count(distinct a,b), count(distinct b,c), count(distinct c,d), count(distinct a,d), count(distinct a,c), count(distinct b,d), count(distinct a,b,c), count(distinct b,c,d), count(distinct a,c,d), count(distinct a,b,d), count(distinct a,b,c,d) from abcd @@ -184,7 +186,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -236,10 +238,10 @@ POSTHOOK: query: select a, count(distinct b), count(distinct c), sum(d) from abc POSTHOOK: type: QUERY POSTHOOK: Input: default@abcd #### A masked pattern was here #### -NULL 1 1 6 10 2 2 10 -12 1 2 9 100 1 1 3 +12 1 2 9 +NULL 1 1 6 PREHOOK: query: explain select count(1), count(*), count(a), count(b), count(c), count(d), count(distinct a), count(distinct b), count(distinct c), count(distinct d), count(distinct a,b), count(distinct b,c), count(distinct c,d), count(distinct a,d), count(distinct a,c), count(distinct b,d), count(distinct a,b,c), count(distinct b,c,d), count(distinct a,c,d), count(distinct a,b,d), count(distinct a,b,c,d) from abcd PREHOOK: type: QUERY POSTHOOK: query: explain select count(1), count(*), count(a), count(b), count(c), count(d), count(distinct a), count(distinct b), count(distinct c), count(distinct d), count(distinct a,b), count(distinct b,c), count(distinct c,d), count(distinct a,d), count(distinct a,c), count(distinct b,d), count(distinct a,b,c), count(distinct b,c,d), count(distinct a,c,d), count(distinct a,b,d), count(distinct a,b,c,d) from abcd diff --git a/ql/src/test/results/clientpositive/spark/ctas.q.out b/ql/src/test/results/clientpositive/spark/ctas.q.out index 2ee6d9f..15c46ed 100644 --- a/ql/src/test/results/clientpositive/spark/ctas.q.out +++ b/ql/src/test/results/clientpositive/spark/ctas.q.out @@ -176,7 +176,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (SORT, 1) + Reducer 2 <- Map 1 (SORT, 3) Reducer 3 <- Reducer 2 (SORT, 1) #### A masked pattern was here #### Vertices: @@ -320,7 +320,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (SORT, 1) + Reducer 2 <- Map 1 (SORT, 3) Reducer 3 <- Reducer 2 (SORT, 1) #### A masked pattern was here #### Vertices: @@ -529,7 +529,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (SORT, 1) + Reducer 2 <- Map 1 (SORT, 3) Reducer 3 <- Reducer 2 (SORT, 1) #### A masked pattern was here #### Vertices: @@ -716,7 +716,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (SORT, 1) + Reducer 2 <- Map 1 (SORT, 3) Reducer 3 <- Reducer 2 (SORT, 1) #### A masked pattern was here #### Vertices: diff --git a/ql/src/test/results/clientpositive/spark/groupby10.q.out b/ql/src/test/results/clientpositive/spark/groupby10.q.out index 47de08c..31ea336 100644 --- a/ql/src/test/results/clientpositive/spark/groupby10.q.out +++ b/ql/src/test/results/clientpositive/spark/groupby10.q.out @@ -268,10 +268,10 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 5 <- Map 1 (SORT, 1) - Reducer 6 <- Map 1 (SORT, 1) - Reducer 3 <- Reducer 5 (GROUP, 1) - Reducer 4 <- Reducer 6 (GROUP, 1) + Reducer 5 <- Map 1 (SORT, 3) + Reducer 6 <- Map 1 (SORT, 3) + Reducer 3 <- Reducer 5 (GROUP, 3) + Reducer 4 <- Reducer 6 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -483,10 +483,10 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 5 <- Map 1 (SORT, 1) - Reducer 6 <- Map 1 (SORT, 1) - Reducer 3 <- Reducer 5 (GROUP, 1) - Reducer 4 <- Reducer 6 (GROUP, 1) + Reducer 5 <- Map 1 (SORT, 3) + Reducer 6 <- Map 1 (SORT, 3) + Reducer 3 <- Reducer 5 (GROUP, 3) + Reducer 4 <- Reducer 6 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 1 diff --git a/ql/src/test/results/clientpositive/spark/groupby8.q.out b/ql/src/test/results/clientpositive/spark/groupby8.q.out index d943957..c0c0cf2 100644 --- a/ql/src/test/results/clientpositive/spark/groupby8.q.out +++ b/ql/src/test/results/clientpositive/spark/groupby8.q.out @@ -828,10 +828,10 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 5 <- Map 1 (SORT, 1) - Reducer 6 <- Map 1 (SORT, 1) - Reducer 3 <- Reducer 5 (GROUP, 1) - Reducer 4 <- Reducer 6 (GROUP, 1) + Reducer 5 <- Map 1 (SORT, 3) + Reducer 6 <- Map 1 (SORT, 3) + Reducer 3 <- Reducer 5 (GROUP, 3) + Reducer 4 <- Reducer 6 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 1 diff --git a/ql/src/test/results/clientpositive/spark/groupby9.q.out b/ql/src/test/results/clientpositive/spark/groupby9.q.out index 308b777..2707239 100644 --- a/ql/src/test/results/clientpositive/spark/groupby9.q.out +++ b/ql/src/test/results/clientpositive/spark/groupby9.q.out @@ -829,10 +829,10 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 5 <- Map 1 (SORT, 1) - Reducer 6 <- Map 1 (SORT, 1) - Reducer 3 <- Reducer 5 (GROUP, 1) - Reducer 4 <- Reducer 6 (GROUP, 1) + Reducer 5 <- Map 1 (SORT, 3) + Reducer 6 <- Map 1 (SORT, 3) + Reducer 3 <- Reducer 5 (GROUP, 3) + Reducer 4 <- Reducer 6 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -1619,10 +1619,10 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 5 <- Map 1 (SORT, 1) - Reducer 6 <- Map 1 (SORT, 1) - Reducer 3 <- Reducer 5 (GROUP, 1) - Reducer 4 <- Reducer 6 (GROUP, 1) + Reducer 5 <- Map 1 (SORT, 3) + Reducer 6 <- Map 1 (SORT, 3) + Reducer 3 <- Reducer 5 (GROUP, 3) + Reducer 4 <- Reducer 6 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -2409,8 +2409,8 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 2 <- Map 4 (GROUP, 1) - Reducer 3 <- Map 5 (GROUP, 1) + Reducer 2 <- Map 4 (GROUP, 3) + Reducer 3 <- Map 5 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 4 @@ -3196,10 +3196,10 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 5 <- Map 1 (SORT, 1) - Reducer 6 <- Map 1 (SORT, 1) - Reducer 3 <- Reducer 5 (GROUP, 1) - Reducer 4 <- Reducer 6 (GROUP, 1) + Reducer 5 <- Map 1 (SORT, 3) + Reducer 6 <- Map 1 (SORT, 3) + Reducer 3 <- Reducer 5 (GROUP, 3) + Reducer 4 <- Reducer 6 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 1 diff --git a/ql/src/test/results/clientpositive/spark/groupby_cube1.q.out b/ql/src/test/results/clientpositive/spark/groupby_cube1.q.out index 2b6585b..8ed3e67 100644 --- a/ql/src/test/results/clientpositive/spark/groupby_cube1.q.out +++ b/ql/src/test/results/clientpositive/spark/groupby_cube1.q.out @@ -122,7 +122,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -199,8 +199,8 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 1) - Reducer 3 <- Reducer 2 (GROUP PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 3) + Reducer 3 <- Reducer 2 (GROUP PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -304,7 +304,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -404,10 +404,10 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 2 <- Map 6 (GROUP PARTITION-LEVEL SORT, 1) - Reducer 4 <- Map 7 (GROUP PARTITION-LEVEL SORT, 1) - Reducer 3 <- Reducer 2 (GROUP PARTITION-LEVEL SORT, 1) - Reducer 5 <- Reducer 4 (GROUP PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 6 (GROUP PARTITION-LEVEL SORT, 3) + Reducer 4 <- Map 7 (GROUP PARTITION-LEVEL SORT, 3) + Reducer 3 <- Reducer 2 (GROUP PARTITION-LEVEL SORT, 3) + Reducer 5 <- Reducer 4 (GROUP PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 6 diff --git a/ql/src/test/results/clientpositive/spark/groupby_multi_single_reducer3.q.out b/ql/src/test/results/clientpositive/spark/groupby_multi_single_reducer3.q.out index c545d47..fc02ea3 100644 --- a/ql/src/test/results/clientpositive/spark/groupby_multi_single_reducer3.q.out +++ b/ql/src/test/results/clientpositive/spark/groupby_multi_single_reducer3.q.out @@ -225,7 +225,7 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 2 <- Map 1 (SORT, 1) + Reducer 2 <- Map 1 (SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -401,7 +401,7 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 2 <- Map 1 (SORT, 1) + Reducer 2 <- Map 1 (SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -577,7 +577,7 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 2 <- Map 1 (SORT, 1) + Reducer 2 <- Map 1 (SORT, 3) #### A masked pattern was here #### Vertices: Map 1 diff --git a/ql/src/test/results/clientpositive/spark/groupby_position.q.out b/ql/src/test/results/clientpositive/spark/groupby_position.q.out index 8e4e547..6367c43 100644 --- a/ql/src/test/results/clientpositive/spark/groupby_position.q.out +++ b/ql/src/test/results/clientpositive/spark/groupby_position.q.out @@ -238,8 +238,8 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 2 <- Map 4 (GROUP PARTITION-LEVEL SORT, 1) - Reducer 3 <- Map 5 (GROUP PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 4 (GROUP PARTITION-LEVEL SORT, 3) + Reducer 3 <- Map 5 (GROUP PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 4 @@ -438,7 +438,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 1) + Reducer 2 <- Map 1 (GROUP, 3) Reducer 3 <- Reducer 2 (SORT, 1) #### A masked pattern was here #### Vertices: @@ -569,7 +569,7 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 5 <- Map 4 (GROUP, 1) + Reducer 5 <- Map 4 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 4 @@ -618,7 +618,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 3) Reducer 3 <- Reducer 2 (SORT, 1) #### A masked pattern was here #### Vertices: diff --git a/ql/src/test/results/clientpositive/spark/groupby_rollup1.q.out b/ql/src/test/results/clientpositive/spark/groupby_rollup1.q.out index 0563354..8905705 100644 --- a/ql/src/test/results/clientpositive/spark/groupby_rollup1.q.out +++ b/ql/src/test/results/clientpositive/spark/groupby_rollup1.q.out @@ -116,7 +116,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -193,8 +193,8 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 1) - Reducer 3 <- Reducer 2 (GROUP PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 3) + Reducer 3 <- Reducer 2 (GROUP PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -292,7 +292,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -392,10 +392,10 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 2 <- Map 6 (GROUP PARTITION-LEVEL SORT, 1) - Reducer 4 <- Map 7 (GROUP PARTITION-LEVEL SORT, 1) - Reducer 3 <- Reducer 2 (GROUP PARTITION-LEVEL SORT, 1) - Reducer 5 <- Reducer 4 (GROUP PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 6 (GROUP PARTITION-LEVEL SORT, 3) + Reducer 4 <- Map 7 (GROUP PARTITION-LEVEL SORT, 3) + Reducer 3 <- Reducer 2 (GROUP PARTITION-LEVEL SORT, 3) + Reducer 5 <- Reducer 4 (GROUP PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 6 diff --git a/ql/src/test/results/clientpositive/spark/groupby_sort_1_23.q.out b/ql/src/test/results/clientpositive/spark/groupby_sort_1_23.q.out index 9507753..98bb89e 100644 --- a/ql/src/test/results/clientpositive/spark/groupby_sort_1_23.q.out +++ b/ql/src/test/results/clientpositive/spark/groupby_sort_1_23.q.out @@ -297,7 +297,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 1) + Reducer 2 <- Map 1 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -1184,7 +1184,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 1) + Reducer 2 <- Map 1 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -1411,7 +1411,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 1) + Reducer 2 <- Map 1 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -1674,7 +1674,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 1) + Reducer 2 <- Map 1 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -1992,7 +1992,7 @@ STAGE PLANS: columns.types int:int #### A masked pattern was here #### name default.outputtbl1 - numFiles 1 + numFiles 3 numRows 5 rawDataSize 17 serialization.ddl struct outputtbl1 { i32 key, i32 cnt} @@ -2094,7 +2094,7 @@ STAGE PLANS: columns.types int:int #### A masked pattern was here #### name default.outputtbl1 - numFiles 1 + numFiles 3 numRows 5 rawDataSize 17 serialization.ddl struct outputtbl1 { i32 key, i32 cnt} @@ -2179,7 +2179,7 @@ STAGE PLANS: columns.types int:int #### A masked pattern was here #### name default.outputtbl1 - numFiles 1 + numFiles 3 numRows 5 rawDataSize 17 serialization.ddl struct outputtbl1 { i32 key, i32 cnt} @@ -2323,7 +2323,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 4 <- Map 3 (GROUP, 1) + Reducer 4 <- Map 3 (GROUP, 3) Union 2 <- Map 1 (NONE, 0), Reducer 4 (NONE, 0) #### A masked pattern was here #### Vertices: @@ -2886,7 +2886,7 @@ STAGE PLANS: columns.types int:int #### A masked pattern was here #### name default.outputtbl1 - numFiles 2 + numFiles 4 numRows 10 rawDataSize 32 serialization.ddl struct outputtbl1 { i32 key, i32 cnt} @@ -2971,7 +2971,7 @@ STAGE PLANS: columns.types int:int #### A masked pattern was here #### name default.outputtbl1 - numFiles 2 + numFiles 4 numRows 10 rawDataSize 32 serialization.ddl struct outputtbl1 { i32 key, i32 cnt} @@ -3116,7 +3116,7 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 3 <- Map 2 (GROUP, 1) + Reducer 3 <- Map 2 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 2 @@ -3419,7 +3419,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 1) + Reducer 2 <- Map 1 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -3694,7 +3694,7 @@ STAGE PLANS: columns.types int:int:string:int #### A masked pattern was here #### name default.outputtbl4 - numFiles 1 + numFiles 3 numRows 6 rawDataSize 48 serialization.ddl struct outputtbl4 { i32 key1, i32 key2, string key3, i32 cnt} @@ -3777,7 +3777,7 @@ STAGE PLANS: columns.types int:int:string:int #### A masked pattern was here #### name default.outputtbl4 - numFiles 1 + numFiles 3 numRows 6 rawDataSize 48 serialization.ddl struct outputtbl4 { i32 key1, i32 key2, string key3, i32 cnt} diff --git a/ql/src/test/results/clientpositive/spark/groupby_sort_skew_1_23.q.out b/ql/src/test/results/clientpositive/spark/groupby_sort_skew_1_23.q.out index bb2f97f..59db1f1 100644 --- a/ql/src/test/results/clientpositive/spark/groupby_sort_skew_1_23.q.out +++ b/ql/src/test/results/clientpositive/spark/groupby_sort_skew_1_23.q.out @@ -297,8 +297,8 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 1) - Reducer 3 <- Reducer 2 (GROUP, 1) + Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 3) + Reducer 3 <- Reducer 2 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -1202,8 +1202,8 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 1) - Reducer 3 <- Reducer 2 (GROUP, 1) + Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 3) + Reducer 3 <- Reducer 2 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -1447,8 +1447,8 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 1) - Reducer 3 <- Reducer 2 (GROUP, 1) + Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 3) + Reducer 3 <- Reducer 2 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -1728,8 +1728,8 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 1) - Reducer 3 <- Reducer 2 (GROUP, 1) + Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 3) + Reducer 3 <- Reducer 2 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -2064,7 +2064,7 @@ STAGE PLANS: columns.types int:int #### A masked pattern was here #### name default.outputtbl1 - numFiles 1 + numFiles 3 numRows 5 rawDataSize 17 serialization.ddl struct outputtbl1 { i32 key, i32 cnt} @@ -2166,7 +2166,7 @@ STAGE PLANS: columns.types int:int #### A masked pattern was here #### name default.outputtbl1 - numFiles 1 + numFiles 3 numRows 5 rawDataSize 17 serialization.ddl struct outputtbl1 { i32 key, i32 cnt} @@ -2251,7 +2251,7 @@ STAGE PLANS: columns.types int:int #### A masked pattern was here #### name default.outputtbl1 - numFiles 1 + numFiles 3 numRows 5 rawDataSize 17 serialization.ddl struct outputtbl1 { i32 key, i32 cnt} @@ -2395,8 +2395,8 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 4 <- Map 3 (GROUP PARTITION-LEVEL SORT, 1) - Reducer 5 <- Reducer 4 (GROUP, 1) + Reducer 4 <- Map 3 (GROUP PARTITION-LEVEL SORT, 3) + Reducer 5 <- Reducer 4 (GROUP, 3) Union 2 <- Map 1 (NONE, 0), Reducer 5 (NONE, 0) #### A masked pattern was here #### Vertices: @@ -2976,7 +2976,7 @@ STAGE PLANS: columns.types int:int #### A masked pattern was here #### name default.outputtbl1 - numFiles 2 + numFiles 4 numRows 10 rawDataSize 32 serialization.ddl struct outputtbl1 { i32 key, i32 cnt} @@ -3061,7 +3061,7 @@ STAGE PLANS: columns.types int:int #### A masked pattern was here #### name default.outputtbl1 - numFiles 2 + numFiles 4 numRows 10 rawDataSize 32 serialization.ddl struct outputtbl1 { i32 key, i32 cnt} @@ -3206,8 +3206,8 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 3 <- Map 2 (GROUP PARTITION-LEVEL SORT, 1) - Reducer 4 <- Reducer 3 (GROUP, 1) + Reducer 3 <- Map 2 (GROUP PARTITION-LEVEL SORT, 3) + Reducer 4 <- Reducer 3 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 2 @@ -3527,8 +3527,8 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 1) - Reducer 3 <- Reducer 2 (GROUP, 1) + Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 3) + Reducer 3 <- Reducer 2 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -3820,7 +3820,7 @@ STAGE PLANS: columns.types int:int:string:int #### A masked pattern was here #### name default.outputtbl4 - numFiles 1 + numFiles 3 numRows 6 rawDataSize 48 serialization.ddl struct outputtbl4 { i32 key1, i32 key2, string key3, i32 cnt} @@ -3903,7 +3903,7 @@ STAGE PLANS: columns.types int:int:string:int #### A masked pattern was here #### name default.outputtbl4 - numFiles 1 + numFiles 3 numRows 6 rawDataSize 48 serialization.ddl struct outputtbl4 { i32 key1, i32 key2, string key3, i32 cnt} diff --git a/ql/src/test/results/clientpositive/spark/having.q.out b/ql/src/test/results/clientpositive/spark/having.q.out index 5e9f20d..3b80bd1 100644 --- a/ql/src/test/results/clientpositive/spark/having.q.out +++ b/ql/src/test/results/clientpositive/spark/having.q.out @@ -95,7 +95,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 1) + Reducer 2 <- Map 1 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -476,7 +476,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 1) + Reducer 2 <- Map 1 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -748,7 +748,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 1) + Reducer 2 <- Map 1 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -949,7 +949,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 1) + Reducer 2 <- Map 1 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -1221,7 +1221,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 1) + Reducer 2 <- Map 1 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 1 diff --git a/ql/src/test/results/clientpositive/spark/join20.q.out b/ql/src/test/results/clientpositive/spark/join20.q.out index a456fbc..d7c2c7c 100644 --- a/ql/src/test/results/clientpositive/spark/join20.q.out +++ b/ql/src/test/results/clientpositive/spark/join20.q.out @@ -751,7 +751,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 3 <- Map 2 (SORT, 1) + Reducer 3 <- Map 2 (SORT, 3) #### A masked pattern was here #### Vertices: Map 2 diff --git a/ql/src/test/results/clientpositive/spark/join38.q.out b/ql/src/test/results/clientpositive/spark/join38.q.out index d977b4d..a511b03 100644 --- a/ql/src/test/results/clientpositive/spark/join38.q.out +++ b/ql/src/test/results/clientpositive/spark/join38.q.out @@ -79,7 +79,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 3 <- Map 2 (GROUP, 1) + Reducer 3 <- Map 2 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 2 diff --git a/ql/src/test/results/clientpositive/spark/join40.q.out b/ql/src/test/results/clientpositive/spark/join40.q.out index 86c54c1..a4a0dc6 100644 --- a/ql/src/test/results/clientpositive/spark/join40.q.out +++ b/ql/src/test/results/clientpositive/spark/join40.q.out @@ -1843,7 +1843,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 3 <- Map 2 (SORT, 1) + Reducer 3 <- Map 2 (SORT, 3) #### A masked pattern was here #### Vertices: Map 2 @@ -2528,7 +2528,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 3 <- Map 2 (SORT, 1) + Reducer 3 <- Map 2 (SORT, 3) #### A masked pattern was here #### Vertices: Map 2 diff --git a/ql/src/test/results/clientpositive/spark/join_filters_overlap.q.out b/ql/src/test/results/clientpositive/spark/join_filters_overlap.q.out index 07e3601..1a6f157 100644 --- a/ql/src/test/results/clientpositive/spark/join_filters_overlap.q.out +++ b/ql/src/test/results/clientpositive/spark/join_filters_overlap.q.out @@ -1288,7 +1288,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 3 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1), Map 5 (PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 3 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3), Map 5 (PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 diff --git a/ql/src/test/results/clientpositive/spark/join_reorder.q.out b/ql/src/test/results/clientpositive/spark/join_reorder.q.out index abe0996..54522f9 100644 --- a/ql/src/test/results/clientpositive/spark/join_reorder.q.out +++ b/ql/src/test/results/clientpositive/spark/join_reorder.q.out @@ -46,10 +46,14 @@ POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/T3.txt' INTO TABLE T3 POSTHOOK: type: LOAD #### A masked pattern was here #### POSTHOOK: Output: default@t3 -PREHOOK: query: EXPLAIN FROM T1 a JOIN src c ON c.key+1=a.key +PREHOOK: query: -- SORT_QUERY_RESULTS + +EXPLAIN FROM T1 a JOIN src c ON c.key+1=a.key SELECT a.key, a.val, c.key PREHOOK: type: QUERY -POSTHOOK: query: EXPLAIN FROM T1 a JOIN src c ON c.key+1=a.key +POSTHOOK: query: -- SORT_QUERY_RESULTS + +EXPLAIN FROM T1 a JOIN src c ON c.key+1=a.key SELECT a.key, a.val, c.key POSTHOOK: type: QUERY STAGE DEPENDENCIES: @@ -475,9 +479,9 @@ POSTHOOK: Input: default@t2 POSTHOOK: Input: default@t3 #### A masked pattern was here #### 2 2 12 12 +7 NULL 17 17 NULL NULL NULL 14 NULL NULL NULL 16 -7 NULL 17 17 PREHOOK: query: FROM T1 a LEFT OUTER JOIN T2 b ON (b.key=a.key) RIGHT OUTER JOIN T3 c ON (c.val = a.val) @@ -497,9 +501,9 @@ POSTHOOK: Input: default@t2 POSTHOOK: Input: default@t3 #### A masked pattern was here #### 2 2 12 12 +7 NULL 17 17 NULL NULL NULL 14 NULL NULL NULL 16 -7 NULL 17 17 PREHOOK: query: EXPLAIN FROM UNIQUEJOIN PRESERVE T1 a (a.key, a.val), PRESERVE T2 b (b.key, b.val), @@ -520,7 +524,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 3 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 3 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -604,7 +608,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 3 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 3 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -690,15 +694,15 @@ POSTHOOK: Input: default@t3 #### A masked pattern was here #### 1 NULL NULL 2 NULL 2 -NULL 2 NULL 3 3 NULL -NULL 4 4 -NULL 5 NULL -NULL NULL 6 7 NULL 7 8 8 NULL 8 8 NULL 8 NULL NULL +NULL 2 NULL +NULL 4 4 +NULL 5 NULL +NULL NULL 6 PREHOOK: query: FROM UNIQUEJOIN PRESERVE T1 a (a.key, a.val), PRESERVE T2 b (b.key, b.val), @@ -721,12 +725,12 @@ POSTHOOK: Input: default@t3 #### A masked pattern was here #### 1 NULL NULL 2 NULL 2 -NULL 2 NULL 3 3 NULL -NULL 4 4 -NULL 5 NULL -NULL NULL 6 7 NULL 7 8 8 NULL 8 8 NULL 8 NULL NULL +NULL 2 NULL +NULL 4 4 +NULL 5 NULL +NULL NULL 6 diff --git a/ql/src/test/results/clientpositive/spark/limit_pushdown.q.out b/ql/src/test/results/clientpositive/spark/limit_pushdown.q.out index ccc0808..d8443fb 100644 --- a/ql/src/test/results/clientpositive/spark/limit_pushdown.q.out +++ b/ql/src/test/results/clientpositive/spark/limit_pushdown.q.out @@ -173,10 +173,10 @@ POSTHOOK: Input: default@src 98 val_98 98 val_98 PREHOOK: query: explain -select value, sum(key + 1) as sum from src group by value limit 20 +select value, sum(key + 1) as sum from src group by value order by value limit 20 PREHOOK: type: QUERY POSTHOOK: query: explain -select value, sum(key + 1) as sum from src group by value limit 20 +select value, sum(key + 1) as sum from src group by value order by value limit 20 POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage @@ -186,7 +186,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 1) + Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 1) #### A masked pattern was here #### Vertices: Map 1 @@ -240,11 +240,11 @@ STAGE PLANS: Processor Tree: ListSink -PREHOOK: query: select value, sum(key + 1) as sum from src group by value limit 20 +PREHOOK: query: select value, sum(key + 1) as sum from src group by value order by value limit 20 PREHOOK: type: QUERY PREHOOK: Input: default@src #### A masked pattern was here #### -POSTHOOK: query: select value, sum(key + 1) as sum from src group by value limit 20 +POSTHOOK: query: select value, sum(key + 1) as sum from src group by value order by value limit 20 POSTHOOK: type: QUERY POSTHOOK: Input: default@src #### A masked pattern was here #### @@ -382,7 +382,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 1) + Reducer 2 <- Map 1 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -462,10 +462,10 @@ POSTHOOK: Input: default@alltypesorc -16379.0 NULL PREHOOK: query: explain -select ctinyint, count(distinct(cdouble)) from alltypesorc group by ctinyint limit 20 +select ctinyint, count(distinct(cdouble)) from alltypesorc group by ctinyint order by ctinyint limit 20 PREHOOK: type: QUERY POSTHOOK: query: explain -select ctinyint, count(distinct(cdouble)) from alltypesorc group by ctinyint limit 20 +select ctinyint, count(distinct(cdouble)) from alltypesorc group by ctinyint order by ctinyint limit 20 POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage @@ -528,11 +528,11 @@ STAGE PLANS: Processor Tree: ListSink -PREHOOK: query: select ctinyint, count(distinct(cdouble)) from alltypesorc group by ctinyint limit 20 +PREHOOK: query: select ctinyint, count(distinct(cdouble)) from alltypesorc group by ctinyint order by ctinyint limit 20 PREHOOK: type: QUERY PREHOOK: Input: default@alltypesorc #### A masked pattern was here #### -POSTHOOK: query: select ctinyint, count(distinct(cdouble)) from alltypesorc group by ctinyint limit 20 +POSTHOOK: query: select ctinyint, count(distinct(cdouble)) from alltypesorc group by ctinyint order by ctinyint limit 20 POSTHOOK: type: QUERY POSTHOOK: Input: default@alltypesorc #### A masked pattern was here #### @@ -558,11 +558,11 @@ POSTHOOK: Input: default@alltypesorc NULL 2932 PREHOOK: query: -- multi distinct explain -select ctinyint, count(distinct(cstring1)), count(distinct(cstring2)) from alltypesorc group by ctinyint limit 20 +select ctinyint, count(distinct(cstring1)), count(distinct(cstring2)) from alltypesorc group by ctinyint order by ctinyint limit 20 PREHOOK: type: QUERY POSTHOOK: query: -- multi distinct explain -select ctinyint, count(distinct(cstring1)), count(distinct(cstring2)) from alltypesorc group by ctinyint limit 20 +select ctinyint, count(distinct(cstring1)), count(distinct(cstring2)) from alltypesorc group by ctinyint order by ctinyint limit 20 POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage @@ -625,11 +625,11 @@ STAGE PLANS: Processor Tree: ListSink -PREHOOK: query: select ctinyint, count(distinct(cstring1)), count(distinct(cstring2)) from alltypesorc group by ctinyint limit 20 +PREHOOK: query: select ctinyint, count(distinct(cstring1)), count(distinct(cstring2)) from alltypesorc group by ctinyint order by ctinyint limit 20 PREHOOK: type: QUERY PREHOOK: Input: default@alltypesorc #### A masked pattern was here #### -POSTHOOK: query: select ctinyint, count(distinct(cstring1)), count(distinct(cstring2)) from alltypesorc group by ctinyint limit 20 +POSTHOOK: query: select ctinyint, count(distinct(cstring1)), count(distinct(cstring2)) from alltypesorc group by ctinyint order by ctinyint limit 20 POSTHOOK: type: QUERY POSTHOOK: Input: default@alltypesorc #### A masked pattern was here #### @@ -695,7 +695,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 1) + Reducer 2 <- Map 1 (GROUP, 3) Reducer 3 <- Reducer 2 (SORT, 1) #### A masked pattern was here #### Vertices: @@ -870,7 +870,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 4 <- Map 3 (GROUP, 1) + Reducer 4 <- Map 3 (GROUP, 3) Reducer 5 <- Reducer 4 (GROUP, 1) #### A masked pattern was here #### Vertices: @@ -966,11 +966,11 @@ STAGE PLANS: PREHOOK: query: -- map aggregation disabled explain -select value, sum(key) as sum from src group by value limit 20 +select value, sum(key) as sum from src group by value order by value limit 20 PREHOOK: type: QUERY POSTHOOK: query: -- map aggregation disabled explain -select value, sum(key) as sum from src group by value limit 20 +select value, sum(key) as sum from src group by value order by value limit 20 POSTHOOK: type: QUERY STAGE DEPENDENCIES: Stage-1 is a root stage @@ -980,7 +980,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 1) + Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 1) #### A masked pattern was here #### Vertices: Map 1 @@ -1028,20 +1028,23 @@ STAGE PLANS: Processor Tree: ListSink -PREHOOK: query: select value, sum(key) as sum from src group by value limit 20 +PREHOOK: query: select value, sum(key) as sum from src group by value order by value limit 20 PREHOOK: type: QUERY PREHOOK: Input: default@src #### A masked pattern was here #### -POSTHOOK: query: select value, sum(key) as sum from src group by value limit 20 +POSTHOOK: query: select value, sum(key) as sum from src group by value order by value limit 20 POSTHOOK: type: QUERY POSTHOOK: Input: default@src #### A masked pattern was here #### +val_0 0.0 val_10 10.0 val_100 200.0 val_103 206.0 +val_104 208.0 val_105 105.0 val_11 11.0 val_111 111.0 +val_113 226.0 val_114 114.0 val_116 116.0 val_118 236.0 @@ -1049,13 +1052,10 @@ val_119 357.0 val_12 24.0 val_120 240.0 val_125 250.0 +val_126 126.0 val_128 384.0 +val_129 258.0 val_131 131.0 -val_137 137.0 -val_138 414.0 -val_15 15.0 -val_174 174.0 -val_193 193.0 PREHOOK: query: -- flush for order-by explain select key,value,value,value,value,value,value,value,value from src order by key limit 100 @@ -1237,7 +1237,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 1) + Reducer 2 <- Map 1 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -1293,103 +1293,103 @@ POSTHOOK: query: select sum(key) as sum from src group by concat(key,value,value POSTHOOK: type: QUERY POSTHOOK: Input: default@src #### A masked pattern was here #### -0.0 10.0 105.0 -11.0 -111.0 +1107.0 114.0 +1227.0 +1251.0 126.0 -131.0 +1314.0 133.0 136.0 143.0 145.0 -150.0 +1494.0 +15.0 153.0 155.0 -156.0 -157.0 -158.0 -160.0 -162.0 163.0 166.0 17.0 -170.0 -177.0 -178.0 -180.0 -181.0 183.0 -186.0 -189.0 -19.0 -192.0 +1872.0 +194.0 194.0 -196.0 -20.0 -200.0 +1956.0 201.0 202.0 -206.0 +208.0 +210.0 214.0 222.0 226.0 -226.0 -228.0 -24.0 -24.0 -240.0 -242.0 +2345.0 +241.0 250.0 -258.0 +252.0 +262.0 +263.0 268.0 -272.0 -273.0 -274.0 -280.0 -282.0 -298.0 -304.0 -309.0 -316.0 -327.0 +283.0 +291.0 +292.0 +30.0 +310.0 328.0 +33.0 330.0 -348.0 +339.0 +341.0 348.0 350.0 -352.0 -357.0 -358.0 -36.0 -382.0 -384.0 +351.0 +360.0 +379.0 +389.0 390.0 -394.0 -399.0 -400.0 +4.0 406.0 -417.0 +41.0 418.0 +419.0 426.0 -430.0 -432.0 +437.0 442.0 446.0 448.0 -458.0 -459.0 +448.0 +449.0 +457.0 466.0 -469.0 -474.0 -478.0 -489.0 -552.0 -561.0 +467.0 +477.0 +479.0 +484.0 +487.0 +496.0 +497.0 +53.0 +544.0 +562.0 +564.0 +57.0 579.0 -597.0 -622.0 -624.0 -636.0 +618.0 +642.0 +65.0 +662.0 +69.0 +74.0 +77.0 +798.0 +819.0 +82.0 +84.0 +85.0 +858.0 +878.0 +916.0 +918.0 +933.0 +956.0 diff --git a/ql/src/test/results/clientpositive/spark/mapjoin_distinct.q.out b/ql/src/test/results/clientpositive/spark/mapjoin_distinct.q.out index 07b0152..2096a8a 100644 --- a/ql/src/test/results/clientpositive/spark/mapjoin_distinct.q.out +++ b/ql/src/test/results/clientpositive/spark/mapjoin_distinct.q.out @@ -190,7 +190,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 3 <- Map 2 (GROUP, 1) + Reducer 3 <- Map 2 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 2 @@ -326,8 +326,8 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 3 <- Map 2 (GROUP PARTITION-LEVEL SORT, 1) - Reducer 4 <- Reducer 3 (GROUP, 1) + Reducer 3 <- Map 2 (GROUP PARTITION-LEVEL SORT, 3) + Reducer 4 <- Reducer 3 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 2 @@ -470,7 +470,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 3 <- Map 2 (GROUP, 1) + Reducer 3 <- Map 2 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 2 diff --git a/ql/src/test/results/clientpositive/spark/mapjoin_filter_on_outerjoin.q.out b/ql/src/test/results/clientpositive/spark/mapjoin_filter_on_outerjoin.q.out index 287f19d..b5e58d5 100644 --- a/ql/src/test/results/clientpositive/spark/mapjoin_filter_on_outerjoin.q.out +++ b/ql/src/test/results/clientpositive/spark/mapjoin_filter_on_outerjoin.q.out @@ -112,7 +112,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 3 <- Map 2 (SORT, 1) + Reducer 3 <- Map 2 (SORT, 3) #### A masked pattern was here #### Vertices: Map 2 @@ -285,7 +285,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 3 <- Map 2 (SORT, 1) + Reducer 3 <- Map 2 (SORT, 3) #### A masked pattern was here #### Vertices: Map 2 diff --git a/ql/src/test/results/clientpositive/spark/mapjoin_test_outer.q.out b/ql/src/test/results/clientpositive/spark/mapjoin_test_outer.q.out index f458227..4bc10b5 100644 --- a/ql/src/test/results/clientpositive/spark/mapjoin_test_outer.q.out +++ b/ql/src/test/results/clientpositive/spark/mapjoin_test_outer.q.out @@ -289,7 +289,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (SORT, 1) + Reducer 2 <- Map 1 (SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -1131,7 +1131,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (SORT, 1) + Reducer 2 <- Map 1 (SORT, 3) #### A masked pattern was here #### Vertices: Map 1 diff --git a/ql/src/test/results/clientpositive/spark/mapreduce2.q.out b/ql/src/test/results/clientpositive/spark/mapreduce2.q.out index e85410b..7956fd0 100644 --- a/ql/src/test/results/clientpositive/spark/mapreduce2.q.out +++ b/ql/src/test/results/clientpositive/spark/mapreduce2.q.out @@ -101,11 +101,11 @@ POSTHOOK: Lineage: dest1.key SCRIPT [(src)src.FieldSchema(name:key, type:string, POSTHOOK: Lineage: dest1.one SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: dest1.ten SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: dest1.value SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ] -PREHOOK: query: SELECT * FROM (SELECT dest1.* FROM dest1 DISTRIBUTE BY key SORT BY key, ten, one, value) T +PREHOOK: query: SELECT * FROM (SELECT dest1.* FROM dest1 DISTRIBUTE BY key SORT BY key, ten, one, value) T ORDER BY key PREHOOK: type: QUERY PREHOOK: Input: default@dest1 #### A masked pattern was here #### -POSTHOOK: query: SELECT * FROM (SELECT dest1.* FROM dest1 DISTRIBUTE BY key SORT BY key, ten, one, value) T +POSTHOOK: query: SELECT * FROM (SELECT dest1.* FROM dest1 DISTRIBUTE BY key SORT BY key, ten, one, value) T ORDER BY key POSTHOOK: type: QUERY POSTHOOK: Input: default@dest1 #### A masked pattern was here #### diff --git a/ql/src/test/results/clientpositive/spark/multi_insert.q.out b/ql/src/test/results/clientpositive/spark/multi_insert.q.out index ebb3616..941c430 100644 --- a/ql/src/test/results/clientpositive/spark/multi_insert.q.out +++ b/ql/src/test/results/clientpositive/spark/multi_insert.q.out @@ -587,7 +587,7 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 2 <- Map 1 (SORT, 1) + Reducer 2 <- Map 1 (SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -745,7 +745,7 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 2 <- Map 1 (SORT, 1) + Reducer 2 <- Map 1 (SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -903,7 +903,7 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 2 <- Map 1 (SORT, 1) + Reducer 2 <- Map 1 (SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -1061,7 +1061,7 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 2 <- Map 1 (SORT, 1) + Reducer 2 <- Map 1 (SORT, 3) #### A masked pattern was here #### Vertices: Map 1 diff --git a/ql/src/test/results/clientpositive/spark/multi_insert_gby.q.out b/ql/src/test/results/clientpositive/spark/multi_insert_gby.q.out index 8d32943..5bbfe19 100644 --- a/ql/src/test/results/clientpositive/spark/multi_insert_gby.q.out +++ b/ql/src/test/results/clientpositive/spark/multi_insert_gby.q.out @@ -234,7 +234,7 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 2 <- Map 1 (SORT, 1) + Reducer 2 <- Map 1 (SORT, 3) #### A masked pattern was here #### Vertices: Map 1 diff --git a/ql/src/test/results/clientpositive/spark/multi_insert_gby3.q.out b/ql/src/test/results/clientpositive/spark/multi_insert_gby3.q.out index a8f1818..fe29a5a 100644 --- a/ql/src/test/results/clientpositive/spark/multi_insert_gby3.q.out +++ b/ql/src/test/results/clientpositive/spark/multi_insert_gby3.q.out @@ -1,8 +1,10 @@ -PREHOOK: query: create table e1 (key string, keyD double) +PREHOOK: query: -- SORT_QUERY_RESULTS +create table e1 (key string, keyD double) PREHOOK: type: CREATETABLE PREHOOK: Output: database:default PREHOOK: Output: default@e1 -POSTHOOK: query: create table e1 (key string, keyD double) +POSTHOOK: query: -- SORT_QUERY_RESULTS +create table e1 (key string, keyD double) POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@e1 @@ -1605,8 +1607,8 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 2 <- Map 4 (GROUP PARTITION-LEVEL SORT, 1) - Reducer 3 <- Map 5 (GROUP PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 4 (GROUP PARTITION-LEVEL SORT, 3) + Reducer 3 <- Map 5 (GROUP PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 4 @@ -1748,7 +1750,7 @@ STAGE PLANS: Spark Edges: Reducer 2 <- Map 1 (SORT, 1) - Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 1) + Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 diff --git a/ql/src/test/results/clientpositive/spark/multi_insert_lateral_view.q.out b/ql/src/test/results/clientpositive/spark/multi_insert_lateral_view.q.out index 08ad9a0..9c19cdd 100644 --- a/ql/src/test/results/clientpositive/spark/multi_insert_lateral_view.q.out +++ b/ql/src/test/results/clientpositive/spark/multi_insert_lateral_view.q.out @@ -291,8 +291,8 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 2 <- Map 4 (GROUP, 1) - Reducer 3 <- Map 5 (GROUP, 1) + Reducer 2 <- Map 4 (GROUP, 3) + Reducer 3 <- Map 5 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 4 @@ -563,8 +563,8 @@ STAGE PLANS: Stage: Stage-3 Spark Edges: - Reducer 2 <- Map 4 (GROUP, 1) - Reducer 3 <- Map 5 (SORT, 1) + Reducer 2 <- Map 4 (GROUP, 3) + Reducer 3 <- Map 5 (SORT, 3) #### A masked pattern was here #### Vertices: Map 4 @@ -854,9 +854,9 @@ STAGE PLANS: Stage: Stage-3 Spark Edges: - Reducer 2 <- Map 5 (GROUP PARTITION-LEVEL SORT, 1) - Reducer 3 <- Map 6 (GROUP PARTITION-LEVEL SORT, 1) - Reducer 4 <- Map 7 (GROUP PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 5 (GROUP PARTITION-LEVEL SORT, 3) + Reducer 3 <- Map 6 (GROUP PARTITION-LEVEL SORT, 3) + Reducer 4 <- Map 7 (GROUP PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 5 @@ -1224,9 +1224,9 @@ STAGE PLANS: Stage: Stage-4 Spark Edges: - Reducer 2 <- Map 5 (GROUP PARTITION-LEVEL SORT, 1) - Reducer 3 <- Map 6 (GROUP PARTITION-LEVEL SORT, 1) - Reducer 4 <- Map 7 (PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 5 (GROUP PARTITION-LEVEL SORT, 3) + Reducer 3 <- Map 6 (GROUP PARTITION-LEVEL SORT, 3) + Reducer 4 <- Map 7 (PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 5 diff --git a/ql/src/test/results/clientpositive/spark/multi_insert_move_tasks_share_dependencies.q.out b/ql/src/test/results/clientpositive/spark/multi_insert_move_tasks_share_dependencies.q.out index 6a00b3b..ea17fa1 100644 --- a/ql/src/test/results/clientpositive/spark/multi_insert_move_tasks_share_dependencies.q.out +++ b/ql/src/test/results/clientpositive/spark/multi_insert_move_tasks_share_dependencies.q.out @@ -604,7 +604,7 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 2 <- Map 1 (SORT, 1) + Reducer 2 <- Map 1 (SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -766,7 +766,7 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 2 <- Map 1 (SORT, 1) + Reducer 2 <- Map 1 (SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -928,7 +928,7 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 2 <- Map 1 (SORT, 1) + Reducer 2 <- Map 1 (SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -1090,7 +1090,7 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 2 <- Map 1 (SORT, 1) + Reducer 2 <- Map 1 (SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -2417,10 +2417,10 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 5 <- Map 1 (SORT, 1) - Reducer 6 <- Map 1 (SORT, 1) - Reducer 3 <- Reducer 5 (SORT, 1) - Reducer 4 <- Reducer 6 (SORT, 1) + Reducer 5 <- Map 1 (SORT, 3) + Reducer 6 <- Map 1 (SORT, 3) + Reducer 3 <- Reducer 5 (SORT, 3) + Reducer 4 <- Reducer 6 (SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -2556,10 +2556,10 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 5 <- Map 1 (SORT, 1) - Reducer 6 <- Map 1 (SORT, 1) - Reducer 3 <- Reducer 5 (SORT, 1) - Reducer 4 <- Reducer 6 (SORT, 1) + Reducer 5 <- Map 1 (SORT, 3) + Reducer 6 <- Map 1 (SORT, 3) + Reducer 3 <- Reducer 5 (SORT, 3) + Reducer 4 <- Reducer 6 (SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -2695,10 +2695,10 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 5 <- Map 1 (SORT, 1) - Reducer 6 <- Map 1 (SORT, 1) - Reducer 3 <- Reducer 5 (SORT, 1) - Reducer 4 <- Reducer 6 (SORT, 1) + Reducer 5 <- Map 1 (SORT, 3) + Reducer 6 <- Map 1 (SORT, 3) + Reducer 3 <- Reducer 5 (SORT, 3) + Reducer 4 <- Reducer 6 (SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -2834,10 +2834,10 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 5 <- Map 1 (SORT, 1) - Reducer 6 <- Map 1 (SORT, 1) - Reducer 3 <- Reducer 5 (SORT, 1) - Reducer 4 <- Reducer 6 (SORT, 1) + Reducer 5 <- Map 1 (SORT, 3) + Reducer 6 <- Map 1 (SORT, 3) + Reducer 3 <- Reducer 5 (SORT, 3) + Reducer 4 <- Reducer 6 (SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -2981,10 +2981,10 @@ STAGE PLANS: Stage: Stage-4 Spark Edges: - Reducer 5 <- Map 1 (SORT, 1) - Reducer 6 <- Map 1 (SORT, 1) - Reducer 3 <- Reducer 5 (SORT, 1) - Reducer 4 <- Reducer 6 (SORT, 1) + Reducer 5 <- Map 1 (SORT, 3) + Reducer 6 <- Map 1 (SORT, 3) + Reducer 3 <- Reducer 5 (SORT, 3) + Reducer 4 <- Reducer 6 (SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -3231,10 +3231,10 @@ STAGE PLANS: Stage: Stage-4 Spark Edges: - Reducer 5 <- Map 1 (SORT, 1) - Reducer 6 <- Map 1 (SORT, 1) - Reducer 3 <- Reducer 5 (SORT, 1) - Reducer 4 <- Reducer 6 (SORT, 1) + Reducer 5 <- Map 1 (SORT, 3) + Reducer 6 <- Map 1 (SORT, 3) + Reducer 3 <- Reducer 5 (SORT, 3) + Reducer 4 <- Reducer 6 (SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -3481,10 +3481,10 @@ STAGE PLANS: Stage: Stage-4 Spark Edges: - Reducer 5 <- Map 1 (SORT, 1) - Reducer 6 <- Map 1 (SORT, 1) - Reducer 3 <- Reducer 5 (SORT, 1) - Reducer 4 <- Reducer 6 (SORT, 1) + Reducer 5 <- Map 1 (SORT, 3) + Reducer 6 <- Map 1 (SORT, 3) + Reducer 3 <- Reducer 5 (SORT, 3) + Reducer 4 <- Reducer 6 (SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -3731,10 +3731,10 @@ STAGE PLANS: Stage: Stage-4 Spark Edges: - Reducer 5 <- Map 1 (SORT, 1) - Reducer 6 <- Map 1 (SORT, 1) - Reducer 3 <- Reducer 5 (SORT, 1) - Reducer 4 <- Reducer 6 (SORT, 1) + Reducer 5 <- Map 1 (SORT, 3) + Reducer 6 <- Map 1 (SORT, 3) + Reducer 3 <- Reducer 5 (SORT, 3) + Reducer 4 <- Reducer 6 (SORT, 3) #### A masked pattern was here #### Vertices: Map 1 diff --git a/ql/src/test/results/clientpositive/spark/multi_join_union.q.out b/ql/src/test/results/clientpositive/spark/multi_join_union.q.out index 3bec983..e0fe067 100644 --- a/ql/src/test/results/clientpositive/spark/multi_join_union.q.out +++ b/ql/src/test/results/clientpositive/spark/multi_join_union.q.out @@ -1,9 +1,13 @@ -PREHOOK: query: CREATE TABLE src11 as SELECT * FROM src +PREHOOK: query: -- SORT_QUERY_RESULTS + +CREATE TABLE src11 as SELECT * FROM src PREHOOK: type: CREATETABLE_AS_SELECT PREHOOK: Input: default@src PREHOOK: Output: database:default PREHOOK: Output: default@src11 -POSTHOOK: query: CREATE TABLE src11 as SELECT * FROM src +POSTHOOK: query: -- SORT_QUERY_RESULTS + +CREATE TABLE src11 as SELECT * FROM src POSTHOOK: type: CREATETABLE_AS_SELECT POSTHOOK: Input: default@src POSTHOOK: Output: database:default @@ -79,7 +83,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 4 <- Map 5 (PARTITION-LEVEL SORT, 1), Union 3 (PARTITION-LEVEL SORT, 1) + Reducer 4 <- Map 5 (PARTITION-LEVEL SORT, 3), Union 3 (PARTITION-LEVEL SORT, 3) Union 3 <- Map 2 (NONE, 0), Map 6 (NONE, 0) #### A masked pattern was here #### Vertices: diff --git a/ql/src/test/results/clientpositive/spark/outer_join_ppr.q.out b/ql/src/test/results/clientpositive/spark/outer_join_ppr.q.out index 97b6c6c..ad358ad 100644 --- a/ql/src/test/results/clientpositive/spark/outer_join_ppr.q.out +++ b/ql/src/test/results/clientpositive/spark/outer_join_ppr.q.out @@ -1,4 +1,6 @@ -PREHOOK: query: EXPLAIN EXTENDED +PREHOOK: query: -- SORT_QUERY_RESULTS + +EXPLAIN EXTENDED FROM src a FULL OUTER JOIN @@ -7,7 +9,9 @@ PREHOOK: query: EXPLAIN EXTENDED SELECT a.key, a.value, b.key, b.value WHERE a.key > 10 AND a.key < 20 AND b.key > 15 AND b.key < 25 PREHOOK: type: QUERY -POSTHOOK: query: EXPLAIN EXTENDED +POSTHOOK: query: -- SORT_QUERY_RESULTS + +EXPLAIN EXTENDED FROM src a FULL OUTER JOIN @@ -584,7 +588,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 3 (PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 3 (PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 diff --git a/ql/src/test/results/clientpositive/spark/ppd_join4.q.out b/ql/src/test/results/clientpositive/spark/ppd_join4.q.out index 9a8dcf0..71439cd 100644 --- a/ql/src/test/results/clientpositive/spark/ppd_join4.q.out +++ b/ql/src/test/results/clientpositive/spark/ppd_join4.q.out @@ -75,7 +75,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 3 <- Map 2 (SORT, 1) + Reducer 3 <- Map 2 (SORT, 3) #### A masked pattern was here #### Vertices: Map 2 diff --git a/ql/src/test/results/clientpositive/spark/ppd_join_filter.q.out b/ql/src/test/results/clientpositive/spark/ppd_join_filter.q.out index eee3bc7..29acb9b 100644 --- a/ql/src/test/results/clientpositive/spark/ppd_join_filter.q.out +++ b/ql/src/test/results/clientpositive/spark/ppd_join_filter.q.out @@ -511,7 +511,7 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 3 <- Map 2 (GROUP, 1) + Reducer 3 <- Map 2 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 2 @@ -894,7 +894,7 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 3 <- Map 2 (GROUP, 1) + Reducer 3 <- Map 2 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 2 @@ -1273,7 +1273,7 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 3 <- Map 2 (GROUP, 1) + Reducer 3 <- Map 2 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 2 diff --git a/ql/src/test/results/clientpositive/spark/ppd_outer_join3.q.out b/ql/src/test/results/clientpositive/spark/ppd_outer_join3.q.out index 0978c4f..bbd671b 100644 --- a/ql/src/test/results/clientpositive/spark/ppd_outer_join3.q.out +++ b/ql/src/test/results/clientpositive/spark/ppd_outer_join3.q.out @@ -1,4 +1,6 @@ -PREHOOK: query: EXPLAIN +PREHOOK: query: -- SORT_QUERY_RESULTS + +EXPLAIN FROM src a FULL OUTER JOIN @@ -7,7 +9,9 @@ PREHOOK: query: EXPLAIN SELECT a.key, a.value, b.key, b.value WHERE a.key > '10' AND a.key < '20' AND b.key > '15' AND b.key < '25' PREHOOK: type: QUERY -POSTHOOK: query: EXPLAIN +POSTHOOK: query: -- SORT_QUERY_RESULTS + +EXPLAIN FROM src a FULL OUTER JOIN @@ -252,7 +256,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 3 (PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 3 (PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 diff --git a/ql/src/test/results/clientpositive/spark/ppd_transform.q.out b/ql/src/test/results/clientpositive/spark/ppd_transform.q.out index b8c32a8..6ee9f72 100644 --- a/ql/src/test/results/clientpositive/spark/ppd_transform.q.out +++ b/ql/src/test/results/clientpositive/spark/ppd_transform.q.out @@ -206,7 +206,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (SORT, 1) + Reducer 2 <- Map 1 (SORT, 3) #### A masked pattern was here #### Vertices: Map 1 diff --git a/ql/src/test/results/clientpositive/spark/ptf_decimal.q.out b/ql/src/test/results/clientpositive/spark/ptf_decimal.q.out index fd51b53..8494e97 100644 --- a/ql/src/test/results/clientpositive/spark/ptf_decimal.q.out +++ b/ql/src/test/results/clientpositive/spark/ptf_decimal.q.out @@ -1,4 +1,6 @@ -PREHOOK: query: -- 1. aggregate functions with decimal type +PREHOOK: query: -- SORT_QUERY_RESULTS + +-- 1. aggregate functions with decimal type select p_mfgr, p_retailprice, lead(p_retailprice) over (partition by p_mfgr ORDER BY p_name) as c1, @@ -9,7 +11,9 @@ from part PREHOOK: type: QUERY PREHOOK: Input: default@part #### A masked pattern was here #### -POSTHOOK: query: -- 1. aggregate functions with decimal type +POSTHOOK: query: -- SORT_QUERY_RESULTS + +-- 1. aggregate functions with decimal type select p_mfgr, p_retailprice, lead(p_retailprice) over (partition by p_mfgr ORDER BY p_name) as c1, @@ -22,30 +26,30 @@ POSTHOOK: Input: default@part #### A masked pattern was here #### Manufacturer#1 1173.15 1173.15 NULL 1173.15 1173.15 Manufacturer#1 1173.15 1753.76 1173.15 1173.15 1173.15 -Manufacturer#1 1753.76 1602.59 1173.15 1173.15 1753.76 -Manufacturer#1 1602.59 1414.42 1753.76 1173.15 1602.59 Manufacturer#1 1414.42 1632.66 1602.59 1173.15 1414.42 +Manufacturer#1 1602.59 1414.42 1753.76 1173.15 1602.59 Manufacturer#1 1632.66 NULL 1414.42 1173.15 1632.66 +Manufacturer#1 1753.76 1602.59 1173.15 1173.15 1753.76 Manufacturer#2 1690.68 1800.7 NULL 1690.68 1690.68 -Manufacturer#2 1800.7 2031.98 1690.68 1690.68 1800.7 -Manufacturer#2 2031.98 1698.66 1800.7 1690.68 2031.98 Manufacturer#2 1698.66 1701.6 2031.98 1690.68 1698.66 Manufacturer#2 1701.6 NULL 1698.66 1690.68 1701.6 -Manufacturer#3 1671.68 1190.27 NULL 1671.68 1671.68 +Manufacturer#2 1800.7 2031.98 1690.68 1690.68 1800.7 +Manufacturer#2 2031.98 1698.66 1800.7 1690.68 2031.98 Manufacturer#3 1190.27 1410.39 1671.68 1671.68 1190.27 +Manufacturer#3 1337.29 NULL 1922.98 1671.68 1337.29 Manufacturer#3 1410.39 1922.98 1190.27 1671.68 1410.39 +Manufacturer#3 1671.68 1190.27 NULL 1671.68 1671.68 Manufacturer#3 1922.98 1337.29 1410.39 1671.68 1922.98 -Manufacturer#3 1337.29 NULL 1922.98 1671.68 1337.29 -Manufacturer#4 1620.67 1375.42 NULL 1620.67 1620.67 -Manufacturer#4 1375.42 1206.26 1620.67 1620.67 1375.42 Manufacturer#4 1206.26 1844.92 1375.42 1620.67 1206.26 -Manufacturer#4 1844.92 1290.35 1206.26 1620.67 1844.92 Manufacturer#4 1290.35 NULL 1844.92 1620.67 1290.35 -Manufacturer#5 1789.69 1611.66 NULL 1789.69 1789.69 -Manufacturer#5 1611.66 1788.73 1789.69 1789.69 1611.66 -Manufacturer#5 1788.73 1018.1 1611.66 1789.69 1788.73 +Manufacturer#4 1375.42 1206.26 1620.67 1620.67 1375.42 +Manufacturer#4 1620.67 1375.42 NULL 1620.67 1620.67 +Manufacturer#4 1844.92 1290.35 1206.26 1620.67 1844.92 Manufacturer#5 1018.1 1464.48 1788.73 1789.69 1018.1 Manufacturer#5 1464.48 NULL 1018.1 1789.69 1464.48 +Manufacturer#5 1611.66 1788.73 1789.69 1789.69 1611.66 +Manufacturer#5 1788.73 1018.1 1611.66 1789.69 1788.73 +Manufacturer#5 1789.69 1611.66 NULL 1789.69 1789.69 PREHOOK: query: -- 2. ranking functions with decimal type select p_mfgr, p_retailprice, @@ -114,32 +118,32 @@ from part POSTHOOK: type: QUERY POSTHOOK: Input: default@part #### A masked pattern was here #### -Manufacturer#1 1753.76 NULL -Manufacturer#1 1632.66 1753.76 -Manufacturer#1 1602.59 1632.66 -Manufacturer#1 1414.42 1602.59 -Manufacturer#1 1173.15 1414.42 Manufacturer#1 1173.15 1173.15 -Manufacturer#2 2031.98 NULL -Manufacturer#2 1800.7 2031.98 -Manufacturer#2 1701.6 1800.7 -Manufacturer#2 1698.66 1701.6 +Manufacturer#1 1173.15 1414.42 +Manufacturer#1 1414.42 1602.59 +Manufacturer#1 1602.59 1632.66 +Manufacturer#1 1632.66 1753.76 +Manufacturer#1 1753.76 NULL Manufacturer#2 1690.68 1698.66 -Manufacturer#3 1922.98 NULL -Manufacturer#3 1671.68 1922.98 -Manufacturer#3 1410.39 1671.68 -Manufacturer#3 1337.29 1410.39 +Manufacturer#2 1698.66 1701.6 +Manufacturer#2 1701.6 1800.7 +Manufacturer#2 1800.7 2031.98 +Manufacturer#2 2031.98 NULL Manufacturer#3 1190.27 1337.29 -Manufacturer#4 1844.92 NULL -Manufacturer#4 1620.67 1844.92 -Manufacturer#4 1375.42 1620.67 -Manufacturer#4 1290.35 1375.42 +Manufacturer#3 1337.29 1410.39 +Manufacturer#3 1410.39 1671.68 +Manufacturer#3 1671.68 1922.98 +Manufacturer#3 1922.98 NULL Manufacturer#4 1206.26 1290.35 -Manufacturer#5 1789.69 NULL -Manufacturer#5 1788.73 1789.69 -Manufacturer#5 1611.66 1788.73 -Manufacturer#5 1464.48 1611.66 +Manufacturer#4 1290.35 1375.42 +Manufacturer#4 1375.42 1620.67 +Manufacturer#4 1620.67 1844.92 +Manufacturer#4 1844.92 NULL Manufacturer#5 1018.1 1464.48 +Manufacturer#5 1464.48 1611.66 +Manufacturer#5 1611.66 1788.73 +Manufacturer#5 1788.73 1789.69 +Manufacturer#5 1789.69 NULL PREHOOK: query: -- 4. partition by decimal select p_mfgr, p_retailprice, @@ -156,29 +160,29 @@ from part POSTHOOK: type: QUERY POSTHOOK: Input: default@part #### A masked pattern was here #### -Manufacturer#5 1018.1 NULL -Manufacturer#1 1173.15 NULL Manufacturer#1 1173.15 1173.15 -Manufacturer#3 1190.27 NULL -Manufacturer#4 1206.26 NULL -Manufacturer#4 1290.35 NULL -Manufacturer#3 1337.29 NULL -Manufacturer#4 1375.42 NULL -Manufacturer#3 1410.39 NULL +Manufacturer#1 1173.15 NULL Manufacturer#1 1414.42 NULL -Manufacturer#5 1464.48 NULL Manufacturer#1 1602.59 NULL -Manufacturer#5 1611.66 NULL -Manufacturer#4 1620.67 NULL Manufacturer#1 1632.66 NULL -Manufacturer#3 1671.68 NULL +Manufacturer#1 1753.76 NULL Manufacturer#2 1690.68 NULL Manufacturer#2 1698.66 NULL Manufacturer#2 1701.6 NULL -Manufacturer#1 1753.76 NULL -Manufacturer#5 1788.73 NULL -Manufacturer#5 1789.69 NULL Manufacturer#2 1800.7 NULL -Manufacturer#4 1844.92 NULL -Manufacturer#3 1922.98 NULL Manufacturer#2 2031.98 NULL +Manufacturer#3 1190.27 NULL +Manufacturer#3 1337.29 NULL +Manufacturer#3 1410.39 NULL +Manufacturer#3 1671.68 NULL +Manufacturer#3 1922.98 NULL +Manufacturer#4 1206.26 NULL +Manufacturer#4 1290.35 NULL +Manufacturer#4 1375.42 NULL +Manufacturer#4 1620.67 NULL +Manufacturer#4 1844.92 NULL +Manufacturer#5 1018.1 NULL +Manufacturer#5 1464.48 NULL +Manufacturer#5 1611.66 NULL +Manufacturer#5 1788.73 NULL +Manufacturer#5 1789.69 NULL diff --git a/ql/src/test/results/clientpositive/spark/ptf_general_queries.q.out b/ql/src/test/results/clientpositive/spark/ptf_general_queries.q.out index 25068da..a3d6194 100644 --- a/ql/src/test/results/clientpositive/spark/ptf_general_queries.q.out +++ b/ql/src/test/results/clientpositive/spark/ptf_general_queries.q.out @@ -1,4 +1,6 @@ -PREHOOK: query: -- 1. testNoPTFNoWindowing +PREHOOK: query: -- SORT_QUERY_RESULTS + +-- 1. testNoPTFNoWindowing select p_mfgr, p_name, p_size from part distribute by p_mfgr @@ -6,7 +8,9 @@ sort by p_name PREHOOK: type: QUERY PREHOOK: Input: default@part #### A masked pattern was here #### -POSTHOOK: query: -- 1. testNoPTFNoWindowing +POSTHOOK: query: -- SORT_QUERY_RESULTS + +-- 1. testNoPTFNoWindowing select p_mfgr, p_name, p_size from part distribute by p_mfgr @@ -14,31 +18,31 @@ sort by p_name POSTHOOK: type: QUERY POSTHOOK: Input: default@part #### A masked pattern was here #### -Manufacturer#5 almond antique blue firebrick mint 31 Manufacturer#1 almond antique burnished rose metallic 2 Manufacturer#1 almond antique burnished rose metallic 2 -Manufacturer#3 almond antique chartreuse khaki white 17 Manufacturer#1 almond antique chartreuse lavender yellow 34 -Manufacturer#3 almond antique forest lavender goldenrod 14 -Manufacturer#4 almond antique gainsboro frosted violet 10 -Manufacturer#5 almond antique medium spring khaki 6 -Manufacturer#3 almond antique metallic orange dim 19 -Manufacturer#3 almond antique misty red olive 1 -Manufacturer#3 almond antique olive coral navajo 45 Manufacturer#1 almond antique salmon chartreuse burlywood 6 -Manufacturer#5 almond antique sky peru orange 2 +Manufacturer#1 almond aquamarine burnished black steel 28 +Manufacturer#1 almond aquamarine pink moccasin thistle 42 Manufacturer#2 almond antique violet chocolate turquoise 14 -Manufacturer#4 almond antique violet mint lemon 39 Manufacturer#2 almond antique violet turquoise frosted 40 -Manufacturer#1 almond aquamarine burnished black steel 28 -Manufacturer#5 almond aquamarine dodger light gainsboro 46 -Manufacturer#4 almond aquamarine floral ivory bisque 27 Manufacturer#2 almond aquamarine midnight light salmon 2 -Manufacturer#1 almond aquamarine pink moccasin thistle 42 Manufacturer#2 almond aquamarine rose maroon antique 25 Manufacturer#2 almond aquamarine sandy cyan gainsboro 18 +Manufacturer#3 almond antique chartreuse khaki white 17 +Manufacturer#3 almond antique forest lavender goldenrod 14 +Manufacturer#3 almond antique metallic orange dim 19 +Manufacturer#3 almond antique misty red olive 1 +Manufacturer#3 almond antique olive coral navajo 45 +Manufacturer#4 almond antique gainsboro frosted violet 10 +Manufacturer#4 almond antique violet mint lemon 39 +Manufacturer#4 almond aquamarine floral ivory bisque 27 Manufacturer#4 almond aquamarine yellow dodger mint 7 Manufacturer#4 almond azure aquamarine papaya violet 12 +Manufacturer#5 almond antique blue firebrick mint 31 +Manufacturer#5 almond antique medium spring khaki 6 +Manufacturer#5 almond antique sky peru orange 2 +Manufacturer#5 almond aquamarine dodger light gainsboro 46 Manufacturer#5 almond azure blanched chiffon midnight 23 PREHOOK: query: -- 2. testUDAFsNoWindowingNoPTFNoGBY select p_mfgr,p_name, p_retailprice, diff --git a/ql/src/test/results/clientpositive/spark/sample10.q.out b/ql/src/test/results/clientpositive/spark/sample10.q.out index a9574ee..33cfe84 100644 --- a/ql/src/test/results/clientpositive/spark/sample10.q.out +++ b/ql/src/test/results/clientpositive/spark/sample10.q.out @@ -89,7 +89,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 1) + Reducer 2 <- Map 1 (GROUP, 3) Reducer 3 <- Reducer 2 (SORT, 1) #### A masked pattern was here #### Vertices: diff --git a/ql/src/test/results/clientpositive/spark/semijoin.q.out b/ql/src/test/results/clientpositive/spark/semijoin.q.out index 9d527ce..d90d50d 100644 --- a/ql/src/test/results/clientpositive/spark/semijoin.q.out +++ b/ql/src/test/results/clientpositive/spark/semijoin.q.out @@ -158,7 +158,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 3 <- Map 2 (SORT, 1) + Reducer 3 <- Map 2 (SORT, 3) #### A masked pattern was here #### Vertices: Map 2 @@ -272,7 +272,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 3 <- Map 2 (SORT, 1) + Reducer 3 <- Map 2 (SORT, 3) #### A masked pattern was here #### Vertices: Map 2 @@ -388,7 +388,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 3 <- Map 2 (SORT, 1) + Reducer 3 <- Map 2 (SORT, 3) #### A masked pattern was here #### Vertices: Map 2 @@ -496,7 +496,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 3 <- Map 2 (SORT, 1) + Reducer 3 <- Map 2 (SORT, 3) #### A masked pattern was here #### Vertices: Map 2 @@ -615,7 +615,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 3 <- Map 2 (SORT, 1) + Reducer 3 <- Map 2 (SORT, 3) #### A masked pattern was here #### Vertices: Map 2 @@ -726,7 +726,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (SORT, 1) + Reducer 2 <- Map 1 (SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -837,7 +837,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 3 <- Map 2 (SORT, 1) + Reducer 3 <- Map 2 (SORT, 3) #### A masked pattern was here #### Vertices: Map 2 @@ -945,7 +945,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 3 <- Map 2 (SORT, 1) + Reducer 3 <- Map 2 (SORT, 3) #### A masked pattern was here #### Vertices: Map 2 @@ -1058,7 +1058,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 3 <- Map 2 (SORT, 1) + Reducer 3 <- Map 2 (SORT, 3) #### A masked pattern was here #### Vertices: Map 2 @@ -1185,7 +1185,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 3 <- Map 2 (SORT, 1) + Reducer 3 <- Map 2 (SORT, 3) #### A masked pattern was here #### Vertices: Map 2 @@ -1318,7 +1318,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (SORT, 1) + Reducer 2 <- Map 1 (SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -1447,7 +1447,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 3 <- Map 2 (SORT, 1) + Reducer 3 <- Map 2 (SORT, 3) #### A masked pattern was here #### Vertices: Map 2 @@ -1599,7 +1599,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 4 <- Map 3 (SORT, 1) + Reducer 4 <- Map 3 (SORT, 3) #### A masked pattern was here #### Vertices: Map 3 @@ -1742,7 +1742,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 4 <- Map 3 (SORT, 1) + Reducer 4 <- Map 3 (SORT, 3) #### A masked pattern was here #### Vertices: Map 3 @@ -1894,7 +1894,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (SORT, 1) + Reducer 2 <- Map 1 (SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -2002,8 +2002,8 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1), Map 5 (PARTITION-LEVEL SORT, 1) - Reducer 3 <- Reducer 2 (SORT, 1) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3), Map 5 (PARTITION-LEVEL SORT, 3) + Reducer 3 <- Reducer 2 (SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -2185,7 +2185,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 4 <- Map 3 (SORT, 1) + Reducer 4 <- Map 3 (SORT, 3) #### A masked pattern was here #### Vertices: Map 3 @@ -2340,7 +2340,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 3 <- Map 2 (SORT, 1) + Reducer 3 <- Map 2 (SORT, 3) #### A masked pattern was here #### Vertices: Map 2 @@ -2450,8 +2450,8 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1), Map 5 (PARTITION-LEVEL SORT, 1) - Reducer 3 <- Reducer 2 (SORT, 1) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3), Map 5 (PARTITION-LEVEL SORT, 3) + Reducer 3 <- Reducer 2 (SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -2645,7 +2645,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 4 <- Map 3 (SORT, 1) + Reducer 4 <- Map 3 (SORT, 3) #### A masked pattern was here #### Vertices: Map 3 diff --git a/ql/src/test/results/clientpositive/spark/skewjoinopt2.q.out b/ql/src/test/results/clientpositive/spark/skewjoinopt2.q.out index 4cecdf2..b0f0706 100644 --- a/ql/src/test/results/clientpositive/spark/skewjoinopt2.q.out +++ b/ql/src/test/results/clientpositive/spark/skewjoinopt2.q.out @@ -399,7 +399,7 @@ STAGE PLANS: Spark Edges: Union 2 <- Map 1 (NONE, 0), Map 6 (NONE, 0) - Reducer 3 <- Union 2 (GROUP, 1) + Reducer 3 <- Union 2 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -573,7 +573,7 @@ STAGE PLANS: Spark Edges: Union 2 <- Map 1 (NONE, 0), Map 6 (NONE, 0) - Reducer 3 <- Union 2 (GROUP, 1) + Reducer 3 <- Union 2 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 1 diff --git a/ql/src/test/results/clientpositive/spark/skewjoinopt3.q.out b/ql/src/test/results/clientpositive/spark/skewjoinopt3.q.out index 49405da..c35400a 100644 --- a/ql/src/test/results/clientpositive/spark/skewjoinopt3.q.out +++ b/ql/src/test/results/clientpositive/spark/skewjoinopt3.q.out @@ -214,8 +214,8 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1) - Reducer 6 <- Map 5 (PARTITION-LEVEL SORT, 1), Map 7 (PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3) + Reducer 6 <- Map 5 (PARTITION-LEVEL SORT, 3), Map 7 (PARTITION-LEVEL SORT, 3) Union 3 <- Reducer 2 (NONE, 0), Reducer 6 (NONE, 0) #### A masked pattern was here #### Vertices: diff --git a/ql/src/test/results/clientpositive/spark/skewjoinopt9.q.out b/ql/src/test/results/clientpositive/spark/skewjoinopt9.q.out index ecb8a65..1019c2e 100644 --- a/ql/src/test/results/clientpositive/spark/skewjoinopt9.q.out +++ b/ql/src/test/results/clientpositive/spark/skewjoinopt9.q.out @@ -225,7 +225,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 1) + Reducer 2 <- Map 1 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 1 diff --git a/ql/src/test/results/clientpositive/spark/smb_mapjoin_1.q.out b/ql/src/test/results/clientpositive/spark/smb_mapjoin_1.q.out index a09dff8..b9224ec 100644 --- a/ql/src/test/results/clientpositive/spark/smb_mapjoin_1.q.out +++ b/ql/src/test/results/clientpositive/spark/smb_mapjoin_1.q.out @@ -46,10 +46,14 @@ POSTHOOK: query: load data local inpath '../../data/files/smbbucket_3.rc' overwr POSTHOOK: type: LOAD #### A masked pattern was here #### POSTHOOK: Output: default@smb_bucket_3 -PREHOOK: query: explain +PREHOOK: query: -- SORT_QUERY_RESULTS + +explain select /*+mapjoin(a)*/ * from smb_bucket_1 a join smb_bucket_2 b on a.key = b.key PREHOOK: type: QUERY -POSTHOOK: query: explain +POSTHOOK: query: -- SORT_QUERY_RESULTS + +explain select /*+mapjoin(a)*/ * from smb_bucket_1 a join smb_bucket_2 b on a.key = b.key POSTHOOK: type: QUERY STAGE DEPENDENCIES: @@ -219,10 +223,10 @@ POSTHOOK: Input: default@smb_bucket_1 POSTHOOK: Input: default@smb_bucket_2 #### A masked pattern was here #### 1 val_1 NULL NULL +10 val_10 NULL NULL 3 val_3 NULL NULL 4 val_4 NULL NULL 5 val_5 NULL NULL -10 val_10 NULL NULL PREHOOK: query: explain select /*+mapjoin(a)*/ * from smb_bucket_1 a right outer join smb_bucket_2 b on a.key = b.key PREHOOK: type: QUERY @@ -324,7 +328,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 3 (PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 3 (PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -388,10 +392,10 @@ POSTHOOK: Input: default@smb_bucket_1 POSTHOOK: Input: default@smb_bucket_2 #### A masked pattern was here #### 1 val_1 NULL NULL +10 val_10 NULL NULL 3 val_3 NULL NULL 4 val_4 NULL NULL 5 val_5 NULL NULL -10 val_10 NULL NULL NULL NULL 20 val_20 NULL NULL 23 val_23 NULL NULL 25 val_25 @@ -569,10 +573,10 @@ POSTHOOK: Input: default@smb_bucket_1 POSTHOOK: Input: default@smb_bucket_2 #### A masked pattern was here #### 1 val_1 NULL NULL +10 val_10 NULL NULL 3 val_3 NULL NULL 4 val_4 NULL NULL 5 val_5 NULL NULL -10 val_10 NULL NULL PREHOOK: query: explain select /*+mapjoin(b)*/ * from smb_bucket_1 a right outer join smb_bucket_2 b on a.key = b.key PREHOOK: type: QUERY @@ -674,7 +678,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 3 (PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 3 (PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -738,10 +742,10 @@ POSTHOOK: Input: default@smb_bucket_1 POSTHOOK: Input: default@smb_bucket_2 #### A masked pattern was here #### 1 val_1 NULL NULL +10 val_10 NULL NULL 3 val_3 NULL NULL 4 val_4 NULL NULL 5 val_5 NULL NULL -10 val_10 NULL NULL NULL NULL 20 val_20 NULL NULL 23 val_23 NULL NULL 25 val_25 diff --git a/ql/src/test/results/clientpositive/spark/smb_mapjoin_14.q.out b/ql/src/test/results/clientpositive/spark/smb_mapjoin_14.q.out index 4bbc455..fc4d7f0 100644 --- a/ql/src/test/results/clientpositive/spark/smb_mapjoin_14.q.out +++ b/ql/src/test/results/clientpositive/spark/smb_mapjoin_14.q.out @@ -208,7 +208,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 1) + Reducer 2 <- Map 1 (GROUP, 3) Reducer 3 <- Reducer 2 (SORT, 1) #### A masked pattern was here #### Vertices: @@ -367,7 +367,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 3 <- Map 2 (GROUP, 1) + Reducer 3 <- Map 2 (GROUP, 3) Reducer 4 <- Reducer 3 (GROUP, 1) #### A masked pattern was here #### Vertices: diff --git a/ql/src/test/results/clientpositive/spark/smb_mapjoin_17.q.out b/ql/src/test/results/clientpositive/spark/smb_mapjoin_17.q.out index 49b9fda..6c736af 100644 --- a/ql/src/test/results/clientpositive/spark/smb_mapjoin_17.q.out +++ b/ql/src/test/results/clientpositive/spark/smb_mapjoin_17.q.out @@ -781,7 +781,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 3 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1), Map 5 (PARTITION-LEVEL SORT, 1), Map 6 (PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 3 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3), Map 5 (PARTITION-LEVEL SORT, 3), Map 6 (PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 diff --git a/ql/src/test/results/clientpositive/spark/smb_mapjoin_2.q.out b/ql/src/test/results/clientpositive/spark/smb_mapjoin_2.q.out index e12a713..bf8b6de 100644 --- a/ql/src/test/results/clientpositive/spark/smb_mapjoin_2.q.out +++ b/ql/src/test/results/clientpositive/spark/smb_mapjoin_2.q.out @@ -46,10 +46,14 @@ POSTHOOK: query: load data local inpath '../../data/files/smbbucket_3.rc' overwr POSTHOOK: type: LOAD #### A masked pattern was here #### POSTHOOK: Output: default@smb_bucket_3 -PREHOOK: query: explain +PREHOOK: query: -- SORT_QUERY_RESULTS + +explain select /*+mapjoin(a)*/ * from smb_bucket_1 a join smb_bucket_3 b on a.key = b.key PREHOOK: type: QUERY -POSTHOOK: query: explain +POSTHOOK: query: -- SORT_QUERY_RESULTS + +explain select /*+mapjoin(a)*/ * from smb_bucket_1 a join smb_bucket_3 b on a.key = b.key POSTHOOK: type: QUERY STAGE DEPENDENCIES: @@ -135,8 +139,8 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@smb_bucket_1 POSTHOOK: Input: default@smb_bucket_3 #### A masked pattern was here #### -4 val_4 4 val_4 10 val_10 10 val_10 +4 val_4 4 val_4 PREHOOK: query: explain select /*+mapjoin(a)*/ * from smb_bucket_1 a left outer join smb_bucket_3 b on a.key = b.key PREHOOK: type: QUERY @@ -221,10 +225,10 @@ POSTHOOK: Input: default@smb_bucket_1 POSTHOOK: Input: default@smb_bucket_3 #### A masked pattern was here #### 1 val_1 NULL NULL +10 val_10 10 val_10 3 val_3 NULL NULL 4 val_4 4 val_4 5 val_5 NULL NULL -10 val_10 10 val_10 PREHOOK: query: explain select /*+mapjoin(a)*/ * from smb_bucket_1 a right outer join smb_bucket_3 b on a.key = b.key PREHOOK: type: QUERY @@ -308,8 +312,8 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@smb_bucket_1 POSTHOOK: Input: default@smb_bucket_3 #### A masked pattern was here #### -4 val_4 4 val_4 10 val_10 10 val_10 +4 val_4 4 val_4 NULL NULL 17 val_17 NULL NULL 19 val_19 NULL NULL 20 val_20 @@ -328,7 +332,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 3 (PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 3 (PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -392,10 +396,10 @@ POSTHOOK: Input: default@smb_bucket_1 POSTHOOK: Input: default@smb_bucket_3 #### A masked pattern was here #### 1 val_1 NULL NULL +10 val_10 10 val_10 3 val_3 NULL NULL 4 val_4 4 val_4 5 val_5 NULL NULL -10 val_10 10 val_10 NULL NULL 17 val_17 NULL NULL 19 val_19 NULL NULL 20 val_20 @@ -489,8 +493,8 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@smb_bucket_1 POSTHOOK: Input: default@smb_bucket_3 #### A masked pattern was here #### -4 val_4 4 val_4 10 val_10 10 val_10 +4 val_4 4 val_4 PREHOOK: query: explain select /*+mapjoin(b)*/ * from smb_bucket_1 a left outer join smb_bucket_3 b on a.key = b.key PREHOOK: type: QUERY @@ -575,10 +579,10 @@ POSTHOOK: Input: default@smb_bucket_1 POSTHOOK: Input: default@smb_bucket_3 #### A masked pattern was here #### 1 val_1 NULL NULL +10 val_10 10 val_10 3 val_3 NULL NULL 4 val_4 4 val_4 5 val_5 NULL NULL -10 val_10 10 val_10 PREHOOK: query: explain select /*+mapjoin(b)*/ * from smb_bucket_1 a right outer join smb_bucket_3 b on a.key = b.key PREHOOK: type: QUERY @@ -662,8 +666,8 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@smb_bucket_1 POSTHOOK: Input: default@smb_bucket_3 #### A masked pattern was here #### -4 val_4 4 val_4 10 val_10 10 val_10 +4 val_4 4 val_4 NULL NULL 17 val_17 NULL NULL 19 val_19 NULL NULL 20 val_20 @@ -682,7 +686,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 3 (PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 3 (PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -746,10 +750,10 @@ POSTHOOK: Input: default@smb_bucket_1 POSTHOOK: Input: default@smb_bucket_3 #### A masked pattern was here #### 1 val_1 NULL NULL +10 val_10 10 val_10 3 val_3 NULL NULL 4 val_4 4 val_4 5 val_5 NULL NULL -10 val_10 10 val_10 NULL NULL 17 val_17 NULL NULL 19 val_19 NULL NULL 20 val_20 diff --git a/ql/src/test/results/clientpositive/spark/smb_mapjoin_4.q.out b/ql/src/test/results/clientpositive/spark/smb_mapjoin_4.q.out index ca48bdd..3ce287f 100644 --- a/ql/src/test/results/clientpositive/spark/smb_mapjoin_4.q.out +++ b/ql/src/test/results/clientpositive/spark/smb_mapjoin_4.q.out @@ -46,10 +46,14 @@ POSTHOOK: query: load data local inpath '../../data/files/smbbucket_3.rc' overwr POSTHOOK: type: LOAD #### A masked pattern was here #### POSTHOOK: Output: default@smb_bucket_3 -PREHOOK: query: explain +PREHOOK: query: -- SORT_QUERY_RESULTS + +explain select /*+mapjoin(a,b)*/ * from smb_bucket_1 a join smb_bucket_2 b on a.key = b.key join smb_bucket_3 c on b.key=c.key PREHOOK: type: QUERY -POSTHOOK: query: explain +POSTHOOK: query: -- SORT_QUERY_RESULTS + +explain select /*+mapjoin(a,b)*/ * from smb_bucket_1 a join smb_bucket_2 b on a.key = b.key join smb_bucket_3 c on b.key=c.key POSTHOOK: type: QUERY STAGE DEPENDENCIES: @@ -377,10 +381,10 @@ POSTHOOK: Input: default@smb_bucket_2 POSTHOOK: Input: default@smb_bucket_3 #### A masked pattern was here #### 1 val_1 NULL NULL NULL NULL +10 val_10 NULL NULL NULL NULL 3 val_3 NULL NULL NULL NULL 4 val_4 NULL NULL NULL NULL 5 val_5 NULL NULL NULL NULL -10 val_10 NULL NULL NULL NULL PREHOOK: query: explain select /*+mapjoin(a,b)*/ * from smb_bucket_1 a left outer join smb_bucket_2 b on a.key = b.key right outer join smb_bucket_3 c on b.key=c.key PREHOOK: type: QUERY @@ -488,12 +492,12 @@ POSTHOOK: Input: default@smb_bucket_1 POSTHOOK: Input: default@smb_bucket_2 POSTHOOK: Input: default@smb_bucket_3 #### A masked pattern was here #### -NULL NULL NULL NULL 4 val_4 NULL NULL NULL NULL 10 val_10 NULL NULL NULL NULL 17 val_17 NULL NULL NULL NULL 19 val_19 NULL NULL NULL NULL 20 val_20 NULL NULL NULL NULL 23 val_23 +NULL NULL NULL NULL 4 val_4 PREHOOK: query: explain select /*+mapjoin(a,b)*/ * from smb_bucket_1 a left outer join smb_bucket_2 b on a.key = b.key full outer join smb_bucket_3 c on b.key=c.key PREHOOK: type: QUERY @@ -508,7 +512,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 3 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 3 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -587,16 +591,16 @@ POSTHOOK: Input: default@smb_bucket_2 POSTHOOK: Input: default@smb_bucket_3 #### A masked pattern was here #### 1 val_1 NULL NULL NULL NULL +10 val_10 NULL NULL NULL NULL 3 val_3 NULL NULL NULL NULL 4 val_4 NULL NULL NULL NULL -NULL NULL NULL NULL 4 val_4 5 val_5 NULL NULL NULL NULL -10 val_10 NULL NULL NULL NULL NULL NULL NULL NULL 10 val_10 NULL NULL NULL NULL 17 val_17 NULL NULL NULL NULL 19 val_19 NULL NULL NULL NULL 20 val_20 NULL NULL NULL NULL 23 val_23 +NULL NULL NULL NULL 4 val_4 PREHOOK: query: explain select /*+mapjoin(a,b)*/ * from smb_bucket_1 a right outer join smb_bucket_2 b on a.key = b.key join smb_bucket_3 c on b.key=c.key PREHOOK: type: QUERY @@ -924,12 +928,12 @@ POSTHOOK: Input: default@smb_bucket_1 POSTHOOK: Input: default@smb_bucket_2 POSTHOOK: Input: default@smb_bucket_3 #### A masked pattern was here #### -NULL NULL NULL NULL 4 val_4 +NULL NULL 20 val_20 20 val_20 +NULL NULL 23 val_23 23 val_23 NULL NULL NULL NULL 10 val_10 NULL NULL NULL NULL 17 val_17 NULL NULL NULL NULL 19 val_19 -NULL NULL 20 val_20 20 val_20 -NULL NULL 23 val_23 23 val_23 +NULL NULL NULL NULL 4 val_4 PREHOOK: query: explain select /*+mapjoin(a,b)*/ * from smb_bucket_1 a right outer join smb_bucket_2 b on a.key = b.key full outer join smb_bucket_3 c on b.key=c.key PREHOOK: type: QUERY @@ -944,7 +948,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 3 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 3 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -1022,14 +1026,14 @@ POSTHOOK: Input: default@smb_bucket_1 POSTHOOK: Input: default@smb_bucket_2 POSTHOOK: Input: default@smb_bucket_3 #### A masked pattern was here #### -NULL NULL NULL NULL 4 val_4 -NULL NULL NULL NULL 10 val_10 -NULL NULL NULL NULL 17 val_17 -NULL NULL NULL NULL 19 val_19 NULL NULL 20 val_20 20 val_20 NULL NULL 23 val_23 23 val_23 NULL NULL 25 val_25 NULL NULL NULL NULL 30 val_30 NULL NULL +NULL NULL NULL NULL 10 val_10 +NULL NULL NULL NULL 17 val_17 +NULL NULL NULL NULL 19 val_19 +NULL NULL NULL NULL 4 val_4 PREHOOK: query: explain select /*+mapjoin(a,b)*/ * from smb_bucket_1 a full outer join smb_bucket_2 b on a.key = b.key join smb_bucket_3 c on b.key=c.key PREHOOK: type: QUERY @@ -1044,7 +1048,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 3 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 3 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -1138,7 +1142,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 3 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 3 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -1217,10 +1221,10 @@ POSTHOOK: Input: default@smb_bucket_2 POSTHOOK: Input: default@smb_bucket_3 #### A masked pattern was here #### 1 val_1 NULL NULL NULL NULL +10 val_10 NULL NULL NULL NULL 3 val_3 NULL NULL NULL NULL 4 val_4 NULL NULL NULL NULL 5 val_5 NULL NULL NULL NULL -10 val_10 NULL NULL NULL NULL NULL NULL 20 val_20 20 val_20 NULL NULL 23 val_23 23 val_23 NULL NULL 25 val_25 NULL NULL @@ -1239,7 +1243,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 3 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 3 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -1317,12 +1321,12 @@ POSTHOOK: Input: default@smb_bucket_1 POSTHOOK: Input: default@smb_bucket_2 POSTHOOK: Input: default@smb_bucket_3 #### A masked pattern was here #### -NULL NULL NULL NULL 4 val_4 +NULL NULL 20 val_20 20 val_20 +NULL NULL 23 val_23 23 val_23 NULL NULL NULL NULL 10 val_10 NULL NULL NULL NULL 17 val_17 NULL NULL NULL NULL 19 val_19 -NULL NULL 20 val_20 20 val_20 -NULL NULL 23 val_23 23 val_23 +NULL NULL NULL NULL 4 val_4 PREHOOK: query: explain select /*+mapjoin(a,b)*/ * from smb_bucket_1 a full outer join smb_bucket_2 b on a.key = b.key full outer join smb_bucket_3 c on b.key=c.key PREHOOK: type: QUERY @@ -1337,7 +1341,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 3 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 3 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -1416,15 +1420,15 @@ POSTHOOK: Input: default@smb_bucket_2 POSTHOOK: Input: default@smb_bucket_3 #### A masked pattern was here #### 1 val_1 NULL NULL NULL NULL +10 val_10 NULL NULL NULL NULL 3 val_3 NULL NULL NULL NULL 4 val_4 NULL NULL NULL NULL -NULL NULL NULL NULL 4 val_4 5 val_5 NULL NULL NULL NULL -10 val_10 NULL NULL NULL NULL -NULL NULL NULL NULL 10 val_10 -NULL NULL NULL NULL 17 val_17 -NULL NULL NULL NULL 19 val_19 NULL NULL 20 val_20 20 val_20 NULL NULL 23 val_23 23 val_23 NULL NULL 25 val_25 NULL NULL NULL NULL 30 val_30 NULL NULL +NULL NULL NULL NULL 10 val_10 +NULL NULL NULL NULL 17 val_17 +NULL NULL NULL NULL 19 val_19 +NULL NULL NULL NULL 4 val_4 diff --git a/ql/src/test/results/clientpositive/spark/smb_mapjoin_5.q.out b/ql/src/test/results/clientpositive/spark/smb_mapjoin_5.q.out index ffb5c9b..82c42e4 100644 --- a/ql/src/test/results/clientpositive/spark/smb_mapjoin_5.q.out +++ b/ql/src/test/results/clientpositive/spark/smb_mapjoin_5.q.out @@ -46,10 +46,14 @@ POSTHOOK: query: load data local inpath '../../data/files/smbbucket_3.rc' overwr POSTHOOK: type: LOAD #### A masked pattern was here #### POSTHOOK: Output: default@smb_bucket_3 -PREHOOK: query: explain +PREHOOK: query: -- SORT_QUERY_RESULTS + +explain select /*+mapjoin(a,c)*/ * from smb_bucket_1 a join smb_bucket_2 b on a.key = b.key join smb_bucket_3 c on b.key=c.key PREHOOK: type: QUERY -POSTHOOK: query: explain +POSTHOOK: query: -- SORT_QUERY_RESULTS + +explain select /*+mapjoin(a,c)*/ * from smb_bucket_1 a join smb_bucket_2 b on a.key = b.key join smb_bucket_3 c on b.key=c.key POSTHOOK: type: QUERY STAGE DEPENDENCIES: @@ -377,10 +381,10 @@ POSTHOOK: Input: default@smb_bucket_2 POSTHOOK: Input: default@smb_bucket_3 #### A masked pattern was here #### 1 val_1 NULL NULL NULL NULL +10 val_10 NULL NULL NULL NULL 3 val_3 NULL NULL NULL NULL 4 val_4 NULL NULL NULL NULL 5 val_5 NULL NULL NULL NULL -10 val_10 NULL NULL NULL NULL PREHOOK: query: explain select /*+mapjoin(a,c)*/ * from smb_bucket_1 a left outer join smb_bucket_2 b on a.key = b.key right outer join smb_bucket_3 c on b.key=c.key PREHOOK: type: QUERY @@ -488,12 +492,12 @@ POSTHOOK: Input: default@smb_bucket_1 POSTHOOK: Input: default@smb_bucket_2 POSTHOOK: Input: default@smb_bucket_3 #### A masked pattern was here #### -NULL NULL NULL NULL 4 val_4 NULL NULL NULL NULL 10 val_10 NULL NULL NULL NULL 17 val_17 NULL NULL NULL NULL 19 val_19 NULL NULL NULL NULL 20 val_20 NULL NULL NULL NULL 23 val_23 +NULL NULL NULL NULL 4 val_4 PREHOOK: query: explain select /*+mapjoin(a,c)*/ * from smb_bucket_1 a left outer join smb_bucket_2 b on a.key = b.key full outer join smb_bucket_3 c on b.key=c.key PREHOOK: type: QUERY @@ -508,7 +512,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 3 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 3 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -587,16 +591,16 @@ POSTHOOK: Input: default@smb_bucket_2 POSTHOOK: Input: default@smb_bucket_3 #### A masked pattern was here #### 1 val_1 NULL NULL NULL NULL +10 val_10 NULL NULL NULL NULL 3 val_3 NULL NULL NULL NULL 4 val_4 NULL NULL NULL NULL -NULL NULL NULL NULL 4 val_4 5 val_5 NULL NULL NULL NULL -10 val_10 NULL NULL NULL NULL NULL NULL NULL NULL 10 val_10 NULL NULL NULL NULL 17 val_17 NULL NULL NULL NULL 19 val_19 NULL NULL NULL NULL 20 val_20 NULL NULL NULL NULL 23 val_23 +NULL NULL NULL NULL 4 val_4 PREHOOK: query: explain select /*+mapjoin(a,c)*/ * from smb_bucket_1 a right outer join smb_bucket_2 b on a.key = b.key join smb_bucket_3 c on b.key=c.key PREHOOK: type: QUERY @@ -924,12 +928,12 @@ POSTHOOK: Input: default@smb_bucket_1 POSTHOOK: Input: default@smb_bucket_2 POSTHOOK: Input: default@smb_bucket_3 #### A masked pattern was here #### -NULL NULL NULL NULL 4 val_4 +NULL NULL 20 val_20 20 val_20 +NULL NULL 23 val_23 23 val_23 NULL NULL NULL NULL 10 val_10 NULL NULL NULL NULL 17 val_17 NULL NULL NULL NULL 19 val_19 -NULL NULL 20 val_20 20 val_20 -NULL NULL 23 val_23 23 val_23 +NULL NULL NULL NULL 4 val_4 PREHOOK: query: explain select /*+mapjoin(a,c)*/ * from smb_bucket_1 a right outer join smb_bucket_2 b on a.key = b.key full outer join smb_bucket_3 c on b.key=c.key PREHOOK: type: QUERY @@ -944,7 +948,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 3 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 3 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -1022,14 +1026,14 @@ POSTHOOK: Input: default@smb_bucket_1 POSTHOOK: Input: default@smb_bucket_2 POSTHOOK: Input: default@smb_bucket_3 #### A masked pattern was here #### -NULL NULL NULL NULL 4 val_4 -NULL NULL NULL NULL 10 val_10 -NULL NULL NULL NULL 17 val_17 -NULL NULL NULL NULL 19 val_19 NULL NULL 20 val_20 20 val_20 NULL NULL 23 val_23 23 val_23 NULL NULL 25 val_25 NULL NULL NULL NULL 30 val_30 NULL NULL +NULL NULL NULL NULL 10 val_10 +NULL NULL NULL NULL 17 val_17 +NULL NULL NULL NULL 19 val_19 +NULL NULL NULL NULL 4 val_4 PREHOOK: query: explain select /*+mapjoin(a,c)*/ * from smb_bucket_1 a full outer join smb_bucket_2 b on a.key = b.key join smb_bucket_3 c on b.key=c.key PREHOOK: type: QUERY @@ -1044,7 +1048,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 3 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 3 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -1138,7 +1142,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 3 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 3 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -1217,10 +1221,10 @@ POSTHOOK: Input: default@smb_bucket_2 POSTHOOK: Input: default@smb_bucket_3 #### A masked pattern was here #### 1 val_1 NULL NULL NULL NULL +10 val_10 NULL NULL NULL NULL 3 val_3 NULL NULL NULL NULL 4 val_4 NULL NULL NULL NULL 5 val_5 NULL NULL NULL NULL -10 val_10 NULL NULL NULL NULL NULL NULL 20 val_20 20 val_20 NULL NULL 23 val_23 23 val_23 NULL NULL 25 val_25 NULL NULL @@ -1239,7 +1243,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 3 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 3 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -1317,12 +1321,12 @@ POSTHOOK: Input: default@smb_bucket_1 POSTHOOK: Input: default@smb_bucket_2 POSTHOOK: Input: default@smb_bucket_3 #### A masked pattern was here #### -NULL NULL NULL NULL 4 val_4 +NULL NULL 20 val_20 20 val_20 +NULL NULL 23 val_23 23 val_23 NULL NULL NULL NULL 10 val_10 NULL NULL NULL NULL 17 val_17 NULL NULL NULL NULL 19 val_19 -NULL NULL 20 val_20 20 val_20 -NULL NULL 23 val_23 23 val_23 +NULL NULL NULL NULL 4 val_4 PREHOOK: query: explain select /*+mapjoin(a,c)*/ * from smb_bucket_1 a full outer join smb_bucket_2 b on a.key = b.key full outer join smb_bucket_3 c on b.key=c.key PREHOOK: type: QUERY @@ -1337,7 +1341,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 3 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 3 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -1416,15 +1420,15 @@ POSTHOOK: Input: default@smb_bucket_2 POSTHOOK: Input: default@smb_bucket_3 #### A masked pattern was here #### 1 val_1 NULL NULL NULL NULL +10 val_10 NULL NULL NULL NULL 3 val_3 NULL NULL NULL NULL 4 val_4 NULL NULL NULL NULL -NULL NULL NULL NULL 4 val_4 5 val_5 NULL NULL NULL NULL -10 val_10 NULL NULL NULL NULL -NULL NULL NULL NULL 10 val_10 -NULL NULL NULL NULL 17 val_17 -NULL NULL NULL NULL 19 val_19 NULL NULL 20 val_20 20 val_20 NULL NULL 23 val_23 23 val_23 NULL NULL 25 val_25 NULL NULL NULL NULL 30 val_30 NULL NULL +NULL NULL NULL NULL 10 val_10 +NULL NULL NULL NULL 17 val_17 +NULL NULL NULL NULL 19 val_19 +NULL NULL NULL NULL 4 val_4 diff --git a/ql/src/test/results/clientpositive/spark/smb_mapjoin_7.q.out b/ql/src/test/results/clientpositive/spark/smb_mapjoin_7.q.out index 363207a..b040a6a 100644 --- a/ql/src/test/results/clientpositive/spark/smb_mapjoin_7.q.out +++ b/ql/src/test/results/clientpositive/spark/smb_mapjoin_7.q.out @@ -623,7 +623,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 3 (PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 3 (PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 diff --git a/ql/src/test/results/clientpositive/spark/smb_mapjoin_8.q.out b/ql/src/test/results/clientpositive/spark/smb_mapjoin_8.q.out index 802f8396..babaff8 100644 --- a/ql/src/test/results/clientpositive/spark/smb_mapjoin_8.q.out +++ b/ql/src/test/results/clientpositive/spark/smb_mapjoin_8.q.out @@ -1,8 +1,12 @@ -PREHOOK: query: create table smb_bucket_input (key int, value string) stored as rcfile +PREHOOK: query: -- SORT_QUERY_RESULTS + +create table smb_bucket_input (key int, value string) stored as rcfile PREHOOK: type: CREATETABLE PREHOOK: Output: database:default PREHOOK: Output: default@smb_bucket_input -POSTHOOK: query: create table smb_bucket_input (key int, value string) stored as rcfile +POSTHOOK: query: -- SORT_QUERY_RESULTS + +create table smb_bucket_input (key int, value string) stored as rcfile POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@smb_bucket_input @@ -68,11 +72,11 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@smb_bucket4_1 POSTHOOK: Input: default@smb_bucket4_2 #### A masked pattern was here #### -4 val_356 NULL NULL -NULL NULL 484 val_169 2000 val_169 NULL NULL -NULL NULL 3000 val_169 +4 val_356 NULL NULL 4000 val_125 NULL NULL +NULL NULL 3000 val_169 +NULL NULL 484 val_169 NULL NULL 5000 val_125 PREHOOK: query: select /*+mapjoin(b)*/ * from smb_bucket4_1 a full outer join smb_bucket4_2 b on a.key = b.key PREHOOK: type: QUERY @@ -84,11 +88,11 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@smb_bucket4_1 POSTHOOK: Input: default@smb_bucket4_2 #### A masked pattern was here #### -4 val_356 NULL NULL -NULL NULL 484 val_169 2000 val_169 NULL NULL -NULL NULL 3000 val_169 +4 val_356 NULL NULL 4000 val_125 NULL NULL +NULL NULL 3000 val_169 +NULL NULL 484 val_169 NULL NULL 5000 val_125 PREHOOK: query: insert overwrite table smb_bucket4_1 select * from smb_bucket_input where key=2000 or key=4000 PREHOOK: type: QUERY @@ -121,8 +125,8 @@ POSTHOOK: Input: default@smb_bucket4_1 POSTHOOK: Input: default@smb_bucket4_2 #### A masked pattern was here #### 2000 val_169 NULL NULL -NULL NULL 3000 val_169 4000 val_125 NULL NULL +NULL NULL 3000 val_169 NULL NULL 5000 val_125 PREHOOK: query: select /*+mapjoin(b)*/ * from smb_bucket4_1 a full outer join smb_bucket4_2 b on a.key = b.key PREHOOK: type: QUERY @@ -135,8 +139,8 @@ POSTHOOK: Input: default@smb_bucket4_1 POSTHOOK: Input: default@smb_bucket4_2 #### A masked pattern was here #### 2000 val_169 NULL NULL -NULL NULL 3000 val_169 4000 val_125 NULL NULL +NULL NULL 3000 val_169 NULL NULL 5000 val_125 PREHOOK: query: insert overwrite table smb_bucket4_1 select * from smb_bucket_input where key=4000 PREHOOK: type: QUERY diff --git a/ql/src/test/results/clientpositive/spark/subquery_in.q.out b/ql/src/test/results/clientpositive/spark/subquery_in.q.out index 4bb6290..552b5ae 100644 --- a/ql/src/test/results/clientpositive/spark/subquery_in.q.out +++ b/ql/src/test/results/clientpositive/spark/subquery_in.q.out @@ -281,7 +281,7 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3) Reducer 3 <- Reducer 2 (GROUP, 1) #### A masked pattern was here #### Vertices: @@ -441,8 +441,8 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 3 <- Map 2 (PARTITION-LEVEL SORT, 1) - Reducer 4 <- Reducer 3 (GROUP, 1) + Reducer 3 <- Map 2 (PARTITION-LEVEL SORT, 3) + Reducer 4 <- Reducer 3 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 2 @@ -609,7 +609,7 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 3 <- Map 2 (GROUP, 1) + Reducer 3 <- Map 2 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 2 @@ -829,7 +829,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 3 <- Map 2 (GROUP, 1) + Reducer 3 <- Map 2 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 2 diff --git a/ql/src/test/results/clientpositive/spark/union25.q.out b/ql/src/test/results/clientpositive/spark/union25.q.out index abb93b6..b64ec53 100644 --- a/ql/src/test/results/clientpositive/spark/union25.q.out +++ b/ql/src/test/results/clientpositive/spark/union25.q.out @@ -68,8 +68,8 @@ STAGE PLANS: Edges: Union 2 <- Map 1 (NONE, 0), Map 6 (NONE, 0) Union 4 <- Map 7 (NONE, 0), Reducer 3 (NONE, 0) - Reducer 3 <- Union 2 (GROUP, 1) - Reducer 5 <- Union 4 (GROUP, 1) + Reducer 3 <- Union 2 (GROUP, 3) + Reducer 5 <- Union 4 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 1 diff --git a/ql/src/test/results/clientpositive/spark/union33.q.out b/ql/src/test/results/clientpositive/spark/union33.q.out index 0a81d4a..1fbc5b5 100644 --- a/ql/src/test/results/clientpositive/spark/union33.q.out +++ b/ql/src/test/results/clientpositive/spark/union33.q.out @@ -196,9 +196,9 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (GROUP PARTITION-LEVEL SORT, 3) Union 4 <- Map 5 (NONE, 0), Reducer 3 (NONE, 0) - Reducer 3 <- Reducer 2 (GROUP, 1) + Reducer 3 <- Reducer 2 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 1 diff --git a/ql/src/test/results/clientpositive/spark/union_remove_19.q.out b/ql/src/test/results/clientpositive/spark/union_remove_19.q.out index 41daaf6..d6d4add 100644 --- a/ql/src/test/results/clientpositive/spark/union_remove_19.q.out +++ b/ql/src/test/results/clientpositive/spark/union_remove_19.q.out @@ -276,8 +276,8 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 1) - Reducer 5 <- Map 4 (GROUP, 1) + Reducer 2 <- Map 1 (GROUP, 3) + Reducer 5 <- Map 4 (GROUP, 3) Union 3 <- Reducer 2 (NONE, 0), Reducer 5 (NONE, 0) #### A masked pattern was here #### Vertices: @@ -448,8 +448,8 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 1) - Reducer 5 <- Map 4 (GROUP, 1) + Reducer 2 <- Map 1 (GROUP, 3) + Reducer 5 <- Map 4 (GROUP, 3) Union 3 <- Reducer 2 (NONE, 0), Reducer 5 (NONE, 0) #### A masked pattern was here #### Vertices: diff --git a/ql/src/test/results/clientpositive/spark/uniquejoin.q.out b/ql/src/test/results/clientpositive/spark/uniquejoin.q.out index b6d3553..b71d5b1 100644 --- a/ql/src/test/results/clientpositive/spark/uniquejoin.q.out +++ b/ql/src/test/results/clientpositive/spark/uniquejoin.q.out @@ -46,14 +46,18 @@ POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/T3.txt' INTO TABLE T3 POSTHOOK: type: LOAD #### A masked pattern was here #### POSTHOOK: Output: default@t3 -PREHOOK: query: FROM UNIQUEJOIN PRESERVE T1 a (a.key), PRESERVE T2 b (b.key), PRESERVE T3 c (c.key) +PREHOOK: query: -- SORT_QUERY_RESULTS + +FROM UNIQUEJOIN PRESERVE T1 a (a.key), PRESERVE T2 b (b.key), PRESERVE T3 c (c.key) SELECT a.key, b.key, c.key PREHOOK: type: QUERY PREHOOK: Input: default@t1 PREHOOK: Input: default@t2 PREHOOK: Input: default@t3 #### A masked pattern was here #### -POSTHOOK: query: FROM UNIQUEJOIN PRESERVE T1 a (a.key), PRESERVE T2 b (b.key), PRESERVE T3 c (c.key) +POSTHOOK: query: -- SORT_QUERY_RESULTS + +FROM UNIQUEJOIN PRESERVE T1 a (a.key), PRESERVE T2 b (b.key), PRESERVE T3 c (c.key) SELECT a.key, b.key, c.key POSTHOOK: type: QUERY POSTHOOK: Input: default@t1 @@ -63,14 +67,14 @@ POSTHOOK: Input: default@t3 1 NULL NULL 2 2 2 3 3 NULL -NULL 4 4 -NULL 5 NULL -NULL NULL 6 7 NULL 7 8 8 NULL 8 8 NULL 8 8 NULL 8 8 NULL +NULL 4 4 +NULL 5 NULL +NULL NULL 6 PREHOOK: query: FROM UNIQUEJOIN T1 a (a.key), T2 b (b.key), T3 c (c.key) SELECT a.key, b.key, c.key PREHOOK: type: QUERY @@ -119,15 +123,15 @@ POSTHOOK: Input: default@t3 #### A masked pattern was here #### 1 11 NULL NULL NULL NULL 2 12 NULL NULL 2 12 -NULL NULL 2 22 NULL NULL 3 13 3 13 NULL NULL -NULL NULL 4 14 4 14 -NULL NULL 5 15 NULL NULL -NULL NULL NULL NULL 6 16 7 17 NULL NULL 7 17 8 18 8 18 NULL NULL 8 18 8 18 NULL NULL 8 28 NULL NULL NULL NULL +NULL NULL 2 22 NULL NULL +NULL NULL 4 14 4 14 +NULL NULL 5 15 NULL NULL +NULL NULL NULL NULL 6 16 PREHOOK: query: FROM UNIQUEJOIN PRESERVE T1 a (a.key), T2 b (b.key), PRESERVE T3 c (c.key) SELECT a.key, b.key, c.key PREHOOK: type: QUERY @@ -145,13 +149,13 @@ POSTHOOK: Input: default@t3 1 NULL NULL 2 2 2 3 3 NULL -NULL 4 4 -NULL NULL 6 7 NULL 7 8 8 NULL 8 8 NULL 8 8 NULL 8 8 NULL +NULL 4 4 +NULL NULL 6 PREHOOK: query: FROM UNIQUEJOIN PRESERVE T1 a (a.key), T2 b(b.key) SELECT a.key, b.key PREHOOK: type: QUERY diff --git a/ql/src/test/results/clientpositive/spark/vector_cast_constant.q.out b/ql/src/test/results/clientpositive/spark/vector_cast_constant.q.out index 5ff382f..e43a0ca 100644 --- a/ql/src/test/results/clientpositive/spark/vector_cast_constant.q.out +++ b/ql/src/test/results/clientpositive/spark/vector_cast_constant.q.out @@ -119,7 +119,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 1) + Reducer 2 <- Map 1 (GROUP, 3) Reducer 3 <- Reducer 2 (SORT, 1) #### A masked pattern was here #### Vertices: diff --git a/ql/src/test/results/clientpositive/spark/vector_decimal_aggregate.q.out b/ql/src/test/results/clientpositive/spark/vector_decimal_aggregate.q.out index 2fbe31f..c9ba3c89 100644 --- a/ql/src/test/results/clientpositive/spark/vector_decimal_aggregate.q.out +++ b/ql/src/test/results/clientpositive/spark/vector_decimal_aggregate.q.out @@ -16,7 +16,9 @@ POSTHOOK: type: CREATETABLE_AS_SELECT POSTHOOK: Input: default@alltypesorc POSTHOOK: Output: database:default POSTHOOK: Output: default@decimal_vgby -PREHOOK: query: -- First only do simple aggregations that output primitives only +PREHOOK: query: -- SORT_QUERY_RESULTS + +-- First only do simple aggregations that output primitives only EXPLAIN SELECT cint, COUNT(cdecimal1), MAX(cdecimal1), MIN(cdecimal1), SUM(cdecimal1), COUNT(cdecimal2), MAX(cdecimal2), MIN(cdecimal2), SUM(cdecimal2) @@ -24,7 +26,9 @@ EXPLAIN SELECT cint, GROUP BY cint HAVING COUNT(*) > 1 PREHOOK: type: QUERY -POSTHOOK: query: -- First only do simple aggregations that output primitives only +POSTHOOK: query: -- SORT_QUERY_RESULTS + +-- First only do simple aggregations that output primitives only EXPLAIN SELECT cint, COUNT(cdecimal1), MAX(cdecimal1), MIN(cdecimal1), SUM(cdecimal1), COUNT(cdecimal2), MAX(cdecimal2), MIN(cdecimal2), SUM(cdecimal2) @@ -40,7 +44,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 1) + Reducer 2 <- Map 1 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -113,13 +117,13 @@ POSTHOOK: query: SELECT cint, POSTHOOK: type: QUERY POSTHOOK: Input: default@decimal_vgby #### A masked pattern was here #### -762 2 5831542.269248378 1531.2194054054 5833073.4886537834 2 6984454.211097692 1833.9456923076925 6986288.1567899996925 -253665376 1024 9767.0054054054 -9779.5486486487 -347484.0818378374 1024 11697.969230769231 -11712.99230769231 -416182.64030769233089 -3728 6 5831542.269248378 -3367.6517567568 5817556.0411483778 6 6984454.211097692 -4033.445769230769 6967702.8672438458471 -563 2 -515.621072973 -3367.6517567568 -3883.2728297298 2 -617.5607769230769 -4033.445769230769 -4651.0065461538459 -6981 3 5831542.269248378 -515.621072973 5830511.027102432 3 6984454.211097692 -617.5607769230769 6983219.0895438458462 -626923679 1024 9723.4027027027 -9778.9513513514 10541.0525297287 1024 11645.746153846154 -11712.276923076923 12625.04759999997746 +253665376 1024 9767.0054054054 -9779.5486486487 -347484.0818378374 1024 11697.969230769231 -11712.99230769231 -416182.64030769233089 528534767 1024 5831542.269248378 -9777.1594594595 11646372.8607481068 1024 6984454.211097692 -11710.130769230771 13948892.79980307629003 +626923679 1024 9723.4027027027 -9778.9513513514 10541.0525297287 1024 11645.746153846154 -11712.276923076923 12625.04759999997746 +6981 3 5831542.269248378 -515.621072973 5830511.027102432 3 6984454.211097692 -617.5607769230769 6983219.0895438458462 +762 2 5831542.269248378 1531.2194054054 5833073.4886537834 2 6984454.211097692 1833.9456923076925 6986288.1567899996925 NULL 3072 9318.4351351351 -4298.1513513514 5018444.1081079808 3072 11160.715384615385 -5147.907692307693 6010604.3076923073536 PREHOOK: query: -- Now add the others... EXPLAIN SELECT cint, @@ -145,7 +149,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 1) + Reducer 2 <- Map 1 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -217,11 +221,11 @@ POSTHOOK: query: SELECT cint, POSTHOOK: type: QUERY POSTHOOK: Input: default@decimal_vgby #### A masked pattern was here #### -762 2 5831542.269248378 1531.2194054054 5833073.4886537834 2916536.7443268917 2915005.5249214866 4122440.3477364695 2 6984454.211097692 1833.9456923076925 6986288.1567899996925 3493144.07839499984625 3491310.1327026924 4937458.140118758 -253665376 1024 9767.0054054054 -9779.5486486487 -347484.0818378374 -339.33992366976309 5708.9563478862 5711.745967572779 1024 11697.969230769231 -11712.99230769231 -416182.64030769233089 -406.428359675480791885 6837.632716002934 6840.973851172274 -3728 6 5831542.269248378 -3367.6517567568 5817556.0411483778 969592.67352472963333 2174330.2092403853 2381859.406131774 6 6984454.211097692 -4033.445769230769 6967702.8672438458471 1161283.811207307641183333 2604201.2704476737 2852759.5602156054 -563 2 -515.621072973 -3367.6517567568 -3883.2728297298 -1941.6364148649 1426.0153418918999 2016.6902366556308 2 -617.5607769230769 -4033.445769230769 -4651.0065461538459 -2325.50327307692295 1707.9424961538462 2415.395441814127 -6981 3 5831542.269248378 -515.621072973 5830511.027102432 1943503.67570081066667 2749258.455012492 3367140.1929065133 3 6984454.211097692 -617.5607769230769 6983219.0895438458462 2327739.696514615282066667 3292794.4113115156 4032833.0678006653 -626923679 1024 9723.4027027027 -9778.9513513514 10541.0525297287 10.29399661106318 5742.09145323734 5744.897264034267 1024 11645.746153846154 -11712.276923076923 12625.04759999997746 12.329148046874977988 6877.318722794877 6880.679250101603 +253665376 1024 9767.0054054054 -9779.5486486487 -347484.0818378374 -339.33992366976309 5708.9563478862 5711.745967572779 1024 11697.969230769231 -11712.99230769231 -416182.64030769233089 -406.428359675480791885 6837.632716002934 6840.973851172274 528534767 1024 5831542.269248378 -9777.1594594595 11646372.8607481068 11373.41099682432305 257528.92988206653 257654.7686043977 1024 6984454.211097692 -11710.130769230771 13948892.79980307629003 13621.965624807691689482 308443.1074570801 308593.82484083984 +626923679 1024 9723.4027027027 -9778.9513513514 10541.0525297287 10.29399661106318 5742.09145323734 5744.897264034267 1024 11645.746153846154 -11712.276923076923 12625.04759999997746 12.329148046874977988 6877.318722794877 6880.679250101603 +6981 3 5831542.269248378 -515.621072973 5830511.027102432 1943503.67570081066667 2749258.455012492 3367140.1929065133 3 6984454.211097692 -617.5607769230769 6983219.0895438458462 2327739.696514615282066667 3292794.4113115156 4032833.0678006653 +762 2 5831542.269248378 1531.2194054054 5833073.4886537834 2916536.7443268917 2915005.5249214866 4122440.3477364695 2 6984454.211097692 1833.9456923076925 6986288.1567899996925 3493144.07839499984625 3491310.1327026924 4937458.140118758 NULL 3072 9318.4351351351 -4298.1513513514 5018444.1081079808 1633.60810810806667 5695.483082135364 5696.4103077145055 3072 11160.715384615385 -5147.907692307693 6010604.3076923073536 1956.576923076922966667 6821.495748565159 6822.606289190924 diff --git a/ql/src/test/results/clientpositive/spark/vector_distinct_2.q.out b/ql/src/test/results/clientpositive/spark/vector_distinct_2.q.out index 7304a0c..279c61a 100644 --- a/ql/src/test/results/clientpositive/spark/vector_distinct_2.q.out +++ b/ql/src/test/results/clientpositive/spark/vector_distinct_2.q.out @@ -119,7 +119,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 1) + Reducer 2 <- Map 1 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 1 diff --git a/ql/src/test/results/clientpositive/spark/vector_groupby_3.q.out b/ql/src/test/results/clientpositive/spark/vector_groupby_3.q.out index 72cc00f..32bddd3 100644 --- a/ql/src/test/results/clientpositive/spark/vector_groupby_3.q.out +++ b/ql/src/test/results/clientpositive/spark/vector_groupby_3.q.out @@ -119,7 +119,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 1) + Reducer 2 <- Map 1 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 1 diff --git a/ql/src/test/results/clientpositive/spark/vector_mapjoin_reduce.q.out b/ql/src/test/results/clientpositive/spark/vector_mapjoin_reduce.q.out index b6a36d8..1927626 100644 --- a/ql/src/test/results/clientpositive/spark/vector_mapjoin_reduce.q.out +++ b/ql/src/test/results/clientpositive/spark/vector_mapjoin_reduce.q.out @@ -257,7 +257,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 3 <- Map 2 (GROUP, 1) + Reducer 3 <- Map 2 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 2 diff --git a/ql/src/test/results/clientpositive/spark/vector_orderby_5.q.out b/ql/src/test/results/clientpositive/spark/vector_orderby_5.q.out index 6a7c990..0992573 100644 --- a/ql/src/test/results/clientpositive/spark/vector_orderby_5.q.out +++ b/ql/src/test/results/clientpositive/spark/vector_orderby_5.q.out @@ -115,7 +115,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 1) + Reducer 2 <- Map 1 (GROUP, 3) Reducer 3 <- Reducer 2 (SORT, 1) #### A masked pattern was here #### Vertices: diff --git a/ql/src/test/results/clientpositive/spark/vectorization_13.q.out b/ql/src/test/results/clientpositive/spark/vectorization_13.q.out index 923d805..ce10280 100644 --- a/ql/src/test/results/clientpositive/spark/vectorization_13.q.out +++ b/ql/src/test/results/clientpositive/spark/vectorization_13.q.out @@ -28,6 +28,7 @@ WHERE (((cfloat < 3569) AND ((ctimestamp2 != 12) AND (ctinyint < 9763215.5639)))) GROUP BY cboolean1, ctinyint, ctimestamp1, cfloat, cstring1 +ORDER BY cboolean1, ctinyint, ctimestamp1, cfloat, cstring1 LIMIT 40 PREHOOK: type: QUERY POSTHOOK: query: EXPLAIN @@ -60,6 +61,7 @@ WHERE (((cfloat < 3569) AND ((ctimestamp2 != 12) AND (ctinyint < 9763215.5639)))) GROUP BY cboolean1, ctinyint, ctimestamp1, cfloat, cstring1 +ORDER BY cboolean1, ctinyint, ctimestamp1, cfloat, cstring1 LIMIT 40 POSTHOOK: type: QUERY STAGE DEPENDENCIES: @@ -71,6 +73,7 @@ STAGE PLANS: Spark Edges: Reducer 2 <- Map 1 (GROUP, 1) + Reducer 3 <- Reducer 2 (SORT, 1) #### A masked pattern was here #### Vertices: Map 1 @@ -107,19 +110,31 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10 Statistics: Num rows: 1365 Data size: 41904 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: boolean), _col1 (type: tinyint), _col2 (type: timestamp), _col3 (type: float), _col4 (type: string), (- _col1) (type: tinyint), _col5 (type: tinyint), ((- _col1) + _col5) (type: tinyint), _col6 (type: double), (_col6 * ((- _col1) + _col5)) (type: double), (- _col6) (type: double), (79.553 * _col3) (type: double), _col7 (type: double), (- _col6) (type: double), _col8 (type: double), (((- _col1) + _col5) - 10.175) (type: double), (- (- _col6)) (type: double), (-26.28 / (- (- _col6))) (type: double), _col9 (type: float), ((_col6 * ((- _col1) + _col5)) / _col1) (type: double), _col10 (type: tinyint) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20 + expressions: _col0 (type: boolean), _col1 (type: tinyint), (- _col6) (type: double), (79.553 * _col3) (type: double), _col7 (type: double), _col8 (type: double), (((- _col1) + _col5) - 10.175) (type: double), (- (- _col6)) (type: double), (-26.28 / (- (- _col6))) (type: double), _col9 (type: float), ((_col6 * ((- _col1) + _col5)) / _col1) (type: double), _col2 (type: timestamp), _col10 (type: tinyint), _col3 (type: float), _col4 (type: string), (- _col1) (type: tinyint), _col5 (type: tinyint), ((- _col1) + _col5) (type: tinyint), _col6 (type: double), (_col6 * ((- _col1) + _col5)) (type: double) + outputColumnNames: _col0, _col1, _col10, _col11, _col12, _col14, _col15, _col16, _col17, _col18, _col19, _col2, _col20, _col3, _col4, _col5, _col6, _col7, _col8, _col9 Statistics: Num rows: 1365 Data size: 41904 Basic stats: COMPLETE Column stats: NONE - Limit - Number of rows: 40 + Reduce Output Operator + key expressions: _col0 (type: boolean), _col1 (type: tinyint), _col2 (type: timestamp), _col3 (type: float), _col4 (type: string) + sort order: +++++ + Statistics: Num rows: 1365 Data size: 41904 Basic stats: COMPLETE Column stats: NONE + value expressions: _col5 (type: tinyint), _col6 (type: tinyint), _col7 (type: tinyint), _col8 (type: double), _col9 (type: double), _col10 (type: double), _col11 (type: double), _col12 (type: double), _col14 (type: double), _col15 (type: double), _col16 (type: double), _col17 (type: double), _col18 (type: float), _col19 (type: double), _col20 (type: tinyint) + Reducer 3 + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: boolean), KEY.reducesinkkey1 (type: tinyint), KEY.reducesinkkey2 (type: timestamp), KEY.reducesinkkey3 (type: float), KEY.reducesinkkey4 (type: string), VALUE._col0 (type: tinyint), VALUE._col1 (type: tinyint), VALUE._col2 (type: tinyint), VALUE._col3 (type: double), VALUE._col4 (type: double), VALUE._col5 (type: double), VALUE._col6 (type: double), VALUE._col7 (type: double), VALUE._col5 (type: double), VALUE._col8 (type: double), VALUE._col9 (type: double), VALUE._col10 (type: double), VALUE._col11 (type: double), VALUE._col12 (type: float), VALUE._col13 (type: double), VALUE._col14 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20 + Statistics: Num rows: 1365 Data size: 41904 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 40 + Statistics: Num rows: 40 Data size: 1200 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false Statistics: Num rows: 40 Data size: 1200 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 40 Data size: 1200 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized Stage: Stage-0 Fetch Operator @@ -156,6 +171,7 @@ WHERE (((cfloat < 3569) AND ((ctimestamp2 != 12) AND (ctinyint < 9763215.5639)))) GROUP BY cboolean1, ctinyint, ctimestamp1, cfloat, cstring1 +ORDER BY cboolean1, ctinyint, ctimestamp1, cfloat, cstring1 LIMIT 40 PREHOOK: type: QUERY PREHOOK: Input: default@alltypesorc @@ -189,50 +205,51 @@ WHERE (((cfloat < 3569) AND ((ctimestamp2 != 12) AND (ctinyint < 9763215.5639)))) GROUP BY cboolean1, ctinyint, ctimestamp1, cfloat, cstring1 +ORDER BY cboolean1, ctinyint, ctimestamp1, cfloat, cstring1 LIMIT 40 POSTHOOK: type: QUERY POSTHOOK: Input: default@alltypesorc #### A masked pattern was here #### -NULL -31 1969-12-31 16:00:16.059 -31.0 NULL 31 -31 0 -31.0 -0.0 31.0 -2466.143 0.0 31.0 0.0 -10.175 -31.0 0.847741935483871 -31.0 0.0 -31 -true 50 1969-12-31 16:00:16.338 50.0 821UdmGbkEf4j -50 50 0 50.0 0.0 -50.0 3977.6499999999996 0.0 -50.0 0.0 -10.175 50.0 -0.5256000000000001 50.0 0.0 50 -NULL 23 1969-12-31 16:00:11.785 23.0 NULL -23 23 0 23.0 0.0 -23.0 1829.719 0.0 -23.0 0.0 -10.175 23.0 -1.142608695652174 23.0 0.0 23 -true 14 1969-12-31 16:00:11.528 14.0 821UdmGbkEf4j -14 14 0 14.0 0.0 -14.0 1113.742 0.0 -14.0 0.0 -10.175 14.0 -1.8771428571428572 14.0 0.0 14 -true 50 1969-12-31 16:00:11.087 50.0 1cGVWH7n1QU -50 50 0 50.0 0.0 -50.0 3977.6499999999996 0.0 -50.0 0.0 -10.175 50.0 -0.5256000000000001 50.0 0.0 50 -NULL 13 1969-12-31 16:00:12.498 13.0 NULL -13 13 0 13.0 0.0 -13.0 1034.1889999999999 0.0 -13.0 0.0 -10.175 13.0 -2.0215384615384617 13.0 0.0 13 -true 36 1969-12-31 16:00:15.145 36.0 1cGVWH7n1QU -36 36 0 36.0 0.0 -36.0 2863.908 0.0 -36.0 0.0 -10.175 36.0 -0.73 36.0 0.0 36 -true 3 1969-12-31 16:00:14.349 3.0 cvLH6Eat2yFsyy7p -3 3 0 3.0 0.0 -3.0 238.659 0.0 -3.0 0.0 -10.175 3.0 -8.76 3.0 0.0 3 -NULL -4 1969-12-31 16:00:13.237 -4.0 NULL 4 -4 0 -4.0 -0.0 4.0 -318.212 0.0 4.0 0.0 -10.175 -4.0 6.57 -4.0 0.0 -4 -NULL -20 1969-12-31 16:00:13.131 -20.0 NULL 20 -20 0 -20.0 -0.0 20.0 -1591.06 0.0 20.0 0.0 -10.175 -20.0 1.314 -20.0 0.0 -20 -true 30 1969-12-31 16:00:15.705 30.0 821UdmGbkEf4j -30 30 0 30.0 0.0 -30.0 2386.59 0.0 -30.0 0.0 -10.175 30.0 -0.876 30.0 0.0 30 -true -57 1969-12-31 16:00:13.365 -57.0 1cGVWH7n1QU 57 -57 0 -57.0 -0.0 57.0 -4534.521 0.0 57.0 0.0 -10.175 -57.0 0.4610526315789474 -57.0 0.0 -57 -true 40 1969-12-31 16:00:16.188 40.0 1cGVWH7n1QU -40 40 0 40.0 0.0 -40.0 3182.12 0.0 -40.0 0.0 -10.175 40.0 -0.657 40.0 0.0 40 -NULL -21 1969-12-31 16:00:12.544 -21.0 NULL 21 -21 0 -21.0 -0.0 21.0 -1670.6129999999998 0.0 21.0 0.0 -10.175 -21.0 1.2514285714285716 -21.0 0.0 -21 -NULL 7 1969-12-31 16:00:14.872 7.0 NULL -7 7 0 7.0 0.0 -7.0 556.871 0.0 -7.0 0.0 -10.175 7.0 -3.7542857142857144 7.0 0.0 7 -true -34 1969-12-31 16:00:14.043 -34.0 821UdmGbkEf4j 34 -34 0 -34.0 -0.0 34.0 -2704.8019999999997 0.0 34.0 0.0 -10.175 -34.0 0.7729411764705882 -34.0 0.0 -34 -NULL -58 1969-12-31 16:00:15.658 -58.0 NULL 58 -58 0 -58.0 -0.0 58.0 -4614.074 0.0 58.0 0.0 -10.175 -58.0 0.4531034482758621 -58.0 0.0 -58 -true -59 1969-12-31 16:00:11.231 -59.0 821UdmGbkEf4j 59 -59 0 -59.0 -0.0 59.0 -4693.6269999999995 0.0 59.0 0.0 -10.175 -59.0 0.44542372881355935 -59.0 0.0 -59 -NULL 44 1969-12-31 16:00:16.279 44.0 NULL -44 44 0 44.0 0.0 -44.0 3500.332 0.0 -44.0 0.0 -10.175 44.0 -0.5972727272727273 44.0 0.0 44 -true -27 1969-12-31 16:00:14.723 -27.0 1cGVWH7n1QU 27 -27 0 -27.0 -0.0 27.0 -2147.931 0.0 27.0 0.0 -10.175 -27.0 0.9733333333333334 -27.0 0.0 -27 -NULL 0 1969-12-31 16:00:13.396 0.0 NULL 0 0 0 0.0 0.0 -0.0 0.0 0.0 -0.0 0.0 -10.175 0.0 NULL 0.0 NULL 0 -true -25 1969-12-31 16:00:15.516 -25.0 821UdmGbkEf4j 25 -25 0 -25.0 -0.0 25.0 -1988.8249999999998 0.0 25.0 0.0 -10.175 -25.0 1.0512000000000001 -25.0 0.0 -25 -true -40 1969-12-31 16:00:11.833 -40.0 cvLH6Eat2yFsyy7p 40 -40 0 -40.0 -0.0 40.0 -3182.12 0.0 40.0 0.0 -10.175 -40.0 0.657 -40.0 0.0 -40 -NULL 29 1969-12-31 16:00:16.19 29.0 NULL -29 29 0 29.0 0.0 -29.0 2307.037 0.0 -29.0 0.0 -10.175 29.0 -0.9062068965517242 29.0 0.0 29 -NULL -27 1969-12-31 16:00:15.435 -27.0 NULL 27 -27 0 -27.0 -0.0 27.0 -2147.931 0.0 27.0 0.0 -10.175 -27.0 0.9733333333333334 -27.0 0.0 -27 +NULL -64 1969-12-31 16:00:11.912 -64.0 NULL 64 -64 0 -64.0 -0.0 64.0 -5091.392 0.0 64.0 0.0 -10.175 -64.0 0.410625 -64.0 0.0 -64 +NULL -64 1969-12-31 16:00:12.339 -64.0 NULL 64 -64 0 -64.0 -0.0 64.0 -5091.392 0.0 64.0 0.0 -10.175 -64.0 0.410625 -64.0 0.0 -64 +NULL -64 1969-12-31 16:00:13.274 -64.0 NULL 64 -64 0 -64.0 -0.0 64.0 -5091.392 0.0 64.0 0.0 -10.175 -64.0 0.410625 -64.0 0.0 -64 +NULL -63 1969-12-31 16:00:11.946 -63.0 NULL 63 -63 0 -63.0 -0.0 63.0 -5011.839 0.0 63.0 0.0 -10.175 -63.0 0.41714285714285715 -63.0 0.0 -63 +NULL -63 1969-12-31 16:00:12.188 -63.0 NULL 63 -63 0 -63.0 -0.0 63.0 -5011.839 0.0 63.0 0.0 -10.175 -63.0 0.41714285714285715 -63.0 0.0 -63 NULL -63 1969-12-31 16:00:15.436 -63.0 NULL 63 -63 0 -63.0 -0.0 63.0 -5011.839 0.0 63.0 0.0 -10.175 -63.0 0.41714285714285715 -63.0 0.0 -63 -NULL -6 1969-12-31 16:00:11.883 -6.0 NULL 6 -6 0 -6.0 -0.0 6.0 -477.318 0.0 6.0 0.0 -10.175 -6.0 4.38 -6.0 0.0 -6 -true 14 1969-12-31 16:00:11.302 14.0 821UdmGbkEf4j -14 14 0 14.0 0.0 -14.0 1113.742 0.0 -14.0 0.0 -10.175 14.0 -1.8771428571428572 14.0 0.0 14 -true -63 1969-12-31 16:00:14.899 -63.0 cvLH6Eat2yFsyy7p 63 -63 0 -63.0 -0.0 63.0 -5011.839 0.0 63.0 0.0 -10.175 -63.0 0.41714285714285715 -63.0 0.0 -63 -true -13 1969-12-31 16:00:15.223 -13.0 1cGVWH7n1QU 13 -13 0 -13.0 -0.0 13.0 -1034.1889999999999 0.0 13.0 0.0 -10.175 -13.0 2.0215384615384617 -13.0 0.0 -13 -true 49 1969-12-31 16:00:15.541 49.0 cvLH6Eat2yFsyy7p -49 49 0 49.0 0.0 -49.0 3898.0969999999998 0.0 -49.0 0.0 -10.175 49.0 -0.5363265306122449 49.0 0.0 49 -true -60 1969-12-31 16:00:12.291 -60.0 821UdmGbkEf4j 60 -60 0 -60.0 -0.0 60.0 -4773.18 0.0 60.0 0.0 -10.175 -60.0 0.438 -60.0 0.0 -60 -NULL 32 1969-12-31 16:00:14.026 32.0 NULL -32 32 0 32.0 0.0 -32.0 2545.696 0.0 -32.0 0.0 -10.175 32.0 -0.82125 32.0 0.0 32 -true 16 1969-12-31 16:00:15.698 16.0 1cGVWH7n1QU -16 16 0 16.0 0.0 -16.0 1272.848 0.0 -16.0 0.0 -10.175 16.0 -1.6425 16.0 0.0 16 -NULL 57 1969-12-31 16:00:14.011 57.0 NULL -57 57 0 57.0 0.0 -57.0 4534.521 0.0 -57.0 0.0 -10.175 57.0 -0.4610526315789474 57.0 0.0 57 -true 41 1969-12-31 16:00:12.235 41.0 cvLH6Eat2yFsyy7p -41 41 0 41.0 0.0 -41.0 3261.673 0.0 -41.0 0.0 -10.175 41.0 -0.6409756097560976 41.0 0.0 41 -true -64 1969-12-31 16:00:12.857 -64.0 821UdmGbkEf4j 64 -64 0 -64.0 -0.0 64.0 -5091.392 0.0 64.0 0.0 -10.175 -64.0 0.410625 -64.0 0.0 -64 -true 36 1969-12-31 16:00:13.825 36.0 cvLH6Eat2yFsyy7p -36 36 0 36.0 0.0 -36.0 2863.908 0.0 -36.0 0.0 -10.175 36.0 -0.73 36.0 0.0 36 -NULL -15 1969-12-31 16:00:15.186 -15.0 NULL 15 -15 0 -15.0 -0.0 15.0 -1193.295 0.0 15.0 0.0 -10.175 -15.0 1.752 -15.0 0.0 -15 -true -2 1969-12-31 16:00:15.431 -2.0 1cGVWH7n1QU 2 -2 0 -2.0 -0.0 2.0 -159.106 0.0 2.0 0.0 -10.175 -2.0 13.14 -2.0 0.0 -2 +NULL -62 1969-12-31 16:00:12.388 -62.0 NULL 62 -62 0 -62.0 -0.0 62.0 -4932.286 0.0 62.0 0.0 -10.175 -62.0 0.4238709677419355 -62.0 0.0 -62 +NULL -62 1969-12-31 16:00:12.591 -62.0 NULL 62 -62 0 -62.0 -0.0 62.0 -4932.286 0.0 62.0 0.0 -10.175 -62.0 0.4238709677419355 -62.0 0.0 -62 +NULL -62 1969-12-31 16:00:14.154 -62.0 NULL 62 -62 0 -62.0 -0.0 62.0 -4932.286 0.0 62.0 0.0 -10.175 -62.0 0.4238709677419355 -62.0 0.0 -62 +NULL -62 1969-12-31 16:00:14.247 -62.0 NULL 62 -62 0 -62.0 -0.0 62.0 -4932.286 0.0 62.0 0.0 -10.175 -62.0 0.4238709677419355 -62.0 0.0 -62 +NULL -62 1969-12-31 16:00:14.517 -62.0 NULL 62 -62 0 -62.0 -0.0 62.0 -4932.286 0.0 62.0 0.0 -10.175 -62.0 0.4238709677419355 -62.0 0.0 -62 +NULL -62 1969-12-31 16:00:14.965 -62.0 NULL 62 -62 0 -62.0 -0.0 62.0 -4932.286 0.0 62.0 0.0 -10.175 -62.0 0.4238709677419355 -62.0 0.0 -62 +NULL -61 1969-12-31 16:00:11.842 -61.0 NULL 61 -61 0 -61.0 -0.0 61.0 -4852.733 0.0 61.0 0.0 -10.175 -61.0 0.4308196721311476 -61.0 0.0 -61 +NULL -61 1969-12-31 16:00:12.454 -61.0 NULL 61 -61 0 -61.0 -0.0 61.0 -4852.733 0.0 61.0 0.0 -10.175 -61.0 0.4308196721311476 -61.0 0.0 -61 +NULL -61 1969-12-31 16:00:14.192 -61.0 NULL 61 -61 0 -61.0 -0.0 61.0 -4852.733 0.0 61.0 0.0 -10.175 -61.0 0.4308196721311476 -61.0 0.0 -61 +NULL -61 1969-12-31 16:00:16.558 -61.0 NULL 61 -61 0 -61.0 -0.0 61.0 -4852.733 0.0 61.0 0.0 -10.175 -61.0 0.4308196721311476 -61.0 0.0 -61 +NULL -60 1969-12-31 16:00:11.504 -60.0 NULL 60 -60 0 -60.0 -0.0 60.0 -4773.18 0.0 60.0 0.0 -10.175 -60.0 0.438 -60.0 0.0 -60 +NULL -60 1969-12-31 16:00:11.641 -60.0 NULL 60 -60 0 -60.0 -0.0 60.0 -4773.18 0.0 60.0 0.0 -10.175 -60.0 0.438 -60.0 0.0 -60 +NULL -60 1969-12-31 16:00:11.996 -60.0 NULL 60 -60 0 -60.0 -0.0 60.0 -4773.18 0.0 60.0 0.0 -10.175 -60.0 0.438 -60.0 0.0 -60 +NULL -60 1969-12-31 16:00:12.779 -60.0 NULL 60 -60 0 -60.0 -0.0 60.0 -4773.18 0.0 60.0 0.0 -10.175 -60.0 0.438 -60.0 0.0 -60 +NULL -59 1969-12-31 16:00:12.008 -59.0 NULL 59 -59 0 -59.0 -0.0 59.0 -4693.6269999999995 0.0 59.0 0.0 -10.175 -59.0 0.44542372881355935 -59.0 0.0 -59 +NULL -59 1969-12-31 16:00:13.15 -59.0 NULL 59 -59 0 -59.0 -0.0 59.0 -4693.6269999999995 0.0 59.0 0.0 -10.175 -59.0 0.44542372881355935 -59.0 0.0 -59 +NULL -59 1969-12-31 16:00:13.625 -59.0 NULL 59 -59 0 -59.0 -0.0 59.0 -4693.6269999999995 0.0 59.0 0.0 -10.175 -59.0 0.44542372881355935 -59.0 0.0 -59 +NULL -59 1969-12-31 16:00:15.296 -59.0 NULL 59 -59 0 -59.0 -0.0 59.0 -4693.6269999999995 0.0 59.0 0.0 -10.175 -59.0 0.44542372881355935 -59.0 0.0 -59 +NULL -59 1969-12-31 16:00:15.861 -59.0 NULL 59 -59 0 -59.0 -0.0 59.0 -4693.6269999999995 0.0 59.0 0.0 -10.175 -59.0 0.44542372881355935 -59.0 0.0 -59 +NULL -58 1969-12-31 16:00:12.065 -58.0 NULL 58 -58 0 -58.0 -0.0 58.0 -4614.074 0.0 58.0 0.0 -10.175 -58.0 0.4531034482758621 -58.0 0.0 -58 +NULL -58 1969-12-31 16:00:12.683 -58.0 NULL 58 -58 0 -58.0 -0.0 58.0 -4614.074 0.0 58.0 0.0 -10.175 -58.0 0.4531034482758621 -58.0 0.0 -58 +NULL -58 1969-12-31 16:00:12.948 -58.0 NULL 58 -58 0 -58.0 -0.0 58.0 -4614.074 0.0 58.0 0.0 -10.175 -58.0 0.4531034482758621 -58.0 0.0 -58 +NULL -58 1969-12-31 16:00:14.066 -58.0 NULL 58 -58 0 -58.0 -0.0 58.0 -4614.074 0.0 58.0 0.0 -10.175 -58.0 0.4531034482758621 -58.0 0.0 -58 +NULL -58 1969-12-31 16:00:15.658 -58.0 NULL 58 -58 0 -58.0 -0.0 58.0 -4614.074 0.0 58.0 0.0 -10.175 -58.0 0.4531034482758621 -58.0 0.0 -58 +NULL -57 1969-12-31 16:00:11.451 -57.0 NULL 57 -57 0 -57.0 -0.0 57.0 -4534.521 0.0 57.0 0.0 -10.175 -57.0 0.4610526315789474 -57.0 0.0 -57 +NULL -57 1969-12-31 16:00:11.883 -57.0 NULL 57 -57 0 -57.0 -0.0 57.0 -4534.521 0.0 57.0 0.0 -10.175 -57.0 0.4610526315789474 -57.0 0.0 -57 +NULL -57 1969-12-31 16:00:12.626 -57.0 NULL 57 -57 0 -57.0 -0.0 57.0 -4534.521 0.0 57.0 0.0 -10.175 -57.0 0.4610526315789474 -57.0 0.0 -57 +NULL -57 1969-12-31 16:00:13.578 -57.0 NULL 57 -57 0 -57.0 -0.0 57.0 -4534.521 0.0 57.0 0.0 -10.175 -57.0 0.4610526315789474 -57.0 0.0 -57 +NULL -57 1969-12-31 16:00:15.39 -57.0 NULL 57 -57 0 -57.0 -0.0 57.0 -4534.521 0.0 57.0 0.0 -10.175 -57.0 0.4610526315789474 -57.0 0.0 -57 +NULL -56 1969-12-31 16:00:13.602 -56.0 NULL 56 -56 0 -56.0 -0.0 56.0 -4454.968 0.0 56.0 0.0 -10.175 -56.0 0.4692857142857143 -56.0 0.0 -56 +NULL -56 1969-12-31 16:00:13.958 -56.0 NULL 56 -56 0 -56.0 -0.0 56.0 -4454.968 0.0 56.0 0.0 -10.175 -56.0 0.4692857142857143 -56.0 0.0 -56 +NULL -56 1969-12-31 16:00:15.038 -56.0 NULL 56 -56 0 -56.0 -0.0 56.0 -4454.968 0.0 56.0 0.0 -10.175 -56.0 0.4692857142857143 -56.0 0.0 -56 +NULL -55 1969-12-31 16:00:11.38 -55.0 NULL 55 -55 0 -55.0 -0.0 55.0 -4375.415 0.0 55.0 0.0 -10.175 -55.0 0.47781818181818186 -55.0 0.0 -55 +NULL -55 1969-12-31 16:00:11.751 -55.0 NULL 55 -55 0 -55.0 -0.0 55.0 -4375.415 0.0 55.0 0.0 -10.175 -55.0 0.47781818181818186 -55.0 0.0 -55 PREHOOK: query: -- double compare timestamp EXPLAIN SELECT cboolean1, @@ -264,6 +281,7 @@ WHERE (((cfloat < 3569) AND ((ctimestamp2 != -1.3359999999999999) AND (ctinyint < 9763215.5639)))) GROUP BY cboolean1, ctinyint, ctimestamp1, cfloat, cstring1 +ORDER BY cboolean1, ctinyint, ctimestamp1, cfloat, cstring1 LIMIT 40 PREHOOK: type: QUERY POSTHOOK: query: -- double compare timestamp @@ -297,6 +315,7 @@ WHERE (((cfloat < 3569) AND ((ctimestamp2 != -1.3359999999999999) AND (ctinyint < 9763215.5639)))) GROUP BY cboolean1, ctinyint, ctimestamp1, cfloat, cstring1 +ORDER BY cboolean1, ctinyint, ctimestamp1, cfloat, cstring1 LIMIT 40 POSTHOOK: type: QUERY STAGE DEPENDENCIES: @@ -307,7 +326,8 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 1) + Reducer 2 <- Map 1 (GROUP, 3) + Reducer 3 <- Reducer 2 (SORT, 1) #### A masked pattern was here #### Vertices: Map 1 @@ -344,19 +364,31 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10 Statistics: Num rows: 1365 Data size: 41904 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: boolean), _col1 (type: tinyint), _col2 (type: timestamp), _col3 (type: float), _col4 (type: string), (- _col1) (type: tinyint), _col5 (type: tinyint), ((- _col1) + _col5) (type: tinyint), _col6 (type: double), (_col6 * ((- _col1) + _col5)) (type: double), (- _col6) (type: double), (79.553 * _col3) (type: double), _col7 (type: double), (- _col6) (type: double), _col8 (type: double), (((- _col1) + _col5) - 10.175) (type: double), (- (- _col6)) (type: double), (-26.28 / (- (- _col6))) (type: double), _col9 (type: float), ((_col6 * ((- _col1) + _col5)) / _col1) (type: double), _col10 (type: tinyint) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20 + expressions: _col0 (type: boolean), _col1 (type: tinyint), (- _col6) (type: double), (79.553 * _col3) (type: double), _col7 (type: double), _col8 (type: double), (((- _col1) + _col5) - 10.175) (type: double), (- (- _col6)) (type: double), (-26.28 / (- (- _col6))) (type: double), _col9 (type: float), ((_col6 * ((- _col1) + _col5)) / _col1) (type: double), _col2 (type: timestamp), _col10 (type: tinyint), _col3 (type: float), _col4 (type: string), (- _col1) (type: tinyint), _col5 (type: tinyint), ((- _col1) + _col5) (type: tinyint), _col6 (type: double), (_col6 * ((- _col1) + _col5)) (type: double) + outputColumnNames: _col0, _col1, _col10, _col11, _col12, _col14, _col15, _col16, _col17, _col18, _col19, _col2, _col20, _col3, _col4, _col5, _col6, _col7, _col8, _col9 Statistics: Num rows: 1365 Data size: 41904 Basic stats: COMPLETE Column stats: NONE - Limit - Number of rows: 40 + Reduce Output Operator + key expressions: _col0 (type: boolean), _col1 (type: tinyint), _col2 (type: timestamp), _col3 (type: float), _col4 (type: string) + sort order: +++++ + Statistics: Num rows: 1365 Data size: 41904 Basic stats: COMPLETE Column stats: NONE + value expressions: _col5 (type: tinyint), _col6 (type: tinyint), _col7 (type: tinyint), _col8 (type: double), _col9 (type: double), _col10 (type: double), _col11 (type: double), _col12 (type: double), _col14 (type: double), _col15 (type: double), _col16 (type: double), _col17 (type: double), _col18 (type: float), _col19 (type: double), _col20 (type: tinyint) + Reducer 3 + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: boolean), KEY.reducesinkkey1 (type: tinyint), KEY.reducesinkkey2 (type: timestamp), KEY.reducesinkkey3 (type: float), KEY.reducesinkkey4 (type: string), VALUE._col0 (type: tinyint), VALUE._col1 (type: tinyint), VALUE._col2 (type: tinyint), VALUE._col3 (type: double), VALUE._col4 (type: double), VALUE._col5 (type: double), VALUE._col6 (type: double), VALUE._col7 (type: double), VALUE._col5 (type: double), VALUE._col8 (type: double), VALUE._col9 (type: double), VALUE._col10 (type: double), VALUE._col11 (type: double), VALUE._col12 (type: float), VALUE._col13 (type: double), VALUE._col14 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20 + Statistics: Num rows: 1365 Data size: 41904 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 40 + Statistics: Num rows: 40 Data size: 1200 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false Statistics: Num rows: 40 Data size: 1200 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 40 Data size: 1200 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized Stage: Stage-0 Fetch Operator @@ -393,6 +425,7 @@ WHERE (((cfloat < 3569) AND ((ctimestamp2 != -1.3359999999999999) AND (ctinyint < 9763215.5639)))) GROUP BY cboolean1, ctinyint, ctimestamp1, cfloat, cstring1 +ORDER BY cboolean1, ctinyint, ctimestamp1, cfloat, cstring1 LIMIT 40 PREHOOK: type: QUERY PREHOOK: Input: default@alltypesorc @@ -426,47 +459,48 @@ WHERE (((cfloat < 3569) AND ((ctimestamp2 != -1.3359999999999999) AND (ctinyint < 9763215.5639)))) GROUP BY cboolean1, ctinyint, ctimestamp1, cfloat, cstring1 +ORDER BY cboolean1, ctinyint, ctimestamp1, cfloat, cstring1 LIMIT 40 POSTHOOK: type: QUERY POSTHOOK: Input: default@alltypesorc #### A masked pattern was here #### -NULL -26 1969-12-31 16:00:08.33 -26.0 NULL 26 -26 0 -26.0 -0.0 26.0 -2068.3779999999997 0.0 26.0 0.0 -10.175 -26.0 1.0107692307692309 -26.0 0.0 -26 -NULL -6 1969-12-31 16:00:07.423 -6.0 NULL 6 -6 0 -6.0 -0.0 6.0 -477.318 0.0 6.0 0.0 -10.175 -6.0 4.38 -6.0 0.0 -6 -true 55 1969-12-31 15:59:58.969 55.0 cvLH6Eat2yFsyy7p -55 55 0 55.0 0.0 -55.0 4375.415 0.0 -55.0 0.0 -10.175 55.0 -0.47781818181818186 55.0 0.0 55 -true 14 1969-12-31 16:00:08.209 14.0 1cGVWH7n1QU -14 14 0 14.0 0.0 -14.0 1113.742 0.0 -14.0 0.0 -10.175 14.0 -1.8771428571428572 14.0 0.0 14 -true 48 1969-12-31 16:00:00.923 48.0 1cGVWH7n1QU -48 48 0 48.0 0.0 -48.0 3818.544 0.0 -48.0 0.0 -10.175 48.0 -0.5475 48.0 0.0 48 -true -10 1969-12-31 16:00:00.082 -10.0 1cGVWH7n1QU 10 -10 0 -10.0 -0.0 10.0 -795.53 0.0 10.0 0.0 -10.175 -10.0 2.628 -10.0 0.0 -10 -true -55 1969-12-31 16:00:05.591 -55.0 1cGVWH7n1QU 55 -55 0 -55.0 -0.0 55.0 -4375.415 0.0 55.0 0.0 -10.175 -55.0 0.47781818181818186 -55.0 0.0 -55 -NULL -35 1969-12-31 16:00:04.52 -35.0 NULL 35 -35 0 -35.0 -0.0 35.0 -2784.355 0.0 35.0 0.0 -10.175 -35.0 0.7508571428571429 -35.0 0.0 -35 -NULL -8 1969-12-31 16:00:00.65 -8.0 NULL 8 -8 0 -8.0 -0.0 8.0 -636.424 0.0 8.0 0.0 -10.175 -8.0 3.285 -8.0 0.0 -8 -NULL -20 1969-12-31 16:00:13.131 -20.0 NULL 20 -20 0 -20.0 -0.0 20.0 -1591.06 0.0 20.0 0.0 -10.175 -20.0 1.314 -20.0 0.0 -20 -true 30 1969-12-31 16:00:15.705 30.0 821UdmGbkEf4j -30 30 0 30.0 0.0 -30.0 2386.59 0.0 -30.0 0.0 -10.175 30.0 -0.876 30.0 0.0 30 -true 40 1969-12-31 16:00:16.188 40.0 1cGVWH7n1QU -40 40 0 40.0 0.0 -40.0 3182.12 0.0 -40.0 0.0 -10.175 40.0 -0.657 40.0 0.0 40 -NULL -21 1969-12-31 16:00:12.544 -21.0 NULL 21 -21 0 -21.0 -0.0 21.0 -1670.6129999999998 0.0 21.0 0.0 -10.175 -21.0 1.2514285714285716 -21.0 0.0 -21 -true -34 1969-12-31 16:00:14.043 -34.0 821UdmGbkEf4j 34 -34 0 -34.0 -0.0 34.0 -2704.8019999999997 0.0 34.0 0.0 -10.175 -34.0 0.7729411764705882 -34.0 0.0 -34 -NULL 23 1969-12-31 16:00:08.307 23.0 NULL -23 23 0 23.0 0.0 -23.0 1829.719 0.0 -23.0 0.0 -10.175 23.0 -1.142608695652174 23.0 0.0 23 -true 8 1969-12-31 16:00:10.496 8.0 821UdmGbkEf4j -8 8 0 8.0 0.0 -8.0 636.424 0.0 -8.0 0.0 -10.175 8.0 -3.285 8.0 0.0 8 -NULL -55 1969-12-31 16:00:08.896 -55.0 NULL 55 -55 0 -55.0 -0.0 55.0 -4375.415 0.0 55.0 0.0 -10.175 -55.0 0.47781818181818186 -55.0 0.0 -55 -NULL -57 1969-12-31 16:00:02.707 -57.0 NULL 57 -57 0 -57.0 -0.0 57.0 -4534.521 0.0 57.0 0.0 -10.175 -57.0 0.4610526315789474 -57.0 0.0 -57 -true 1 1969-12-31 16:00:00.125 1.0 cvLH6Eat2yFsyy7p -1 1 0 1.0 0.0 -1.0 79.553 0.0 -1.0 0.0 -10.175 1.0 -26.28 1.0 0.0 1 -true -1 1969-12-31 16:00:00.107 -1.0 1cGVWH7n1QU 1 -1 0 -1.0 -0.0 1.0 -79.553 0.0 1.0 0.0 -10.175 -1.0 26.28 -1.0 0.0 -1 -true -59 1969-12-31 16:00:11.231 -59.0 821UdmGbkEf4j 59 -59 0 -59.0 -0.0 59.0 -4693.6269999999995 0.0 59.0 0.0 -10.175 -59.0 0.44542372881355935 -59.0 0.0 -59 -NULL 33 1969-12-31 16:00:07.006 33.0 NULL -33 33 0 33.0 0.0 -33.0 2625.249 0.0 -33.0 0.0 -10.175 33.0 -0.7963636363636364 33.0 0.0 33 -NULL -58 1969-12-31 16:00:15.658 -58.0 NULL 58 -58 0 -58.0 -0.0 58.0 -4614.074 0.0 58.0 0.0 -10.175 -58.0 0.4531034482758621 -58.0 0.0 -58 -true -6 1969-12-31 16:00:07.372 -6.0 cvLH6Eat2yFsyy7p 6 -6 0 -6.0 -0.0 6.0 -477.318 0.0 6.0 0.0 -10.175 -6.0 4.38 -6.0 0.0 -6 -true -27 1969-12-31 16:00:14.723 -27.0 1cGVWH7n1QU 27 -27 0 -27.0 -0.0 27.0 -2147.931 0.0 27.0 0.0 -10.175 -27.0 0.9733333333333334 -27.0 0.0 -27 -NULL 29 1969-12-31 16:00:16.19 29.0 NULL -29 29 0 29.0 0.0 -29.0 2307.037 0.0 -29.0 0.0 -10.175 29.0 -0.9062068965517242 29.0 0.0 29 -NULL -27 1969-12-31 16:00:15.435 -27.0 NULL 27 -27 0 -27.0 -0.0 27.0 -2147.931 0.0 27.0 0.0 -10.175 -27.0 0.9733333333333334 -27.0 0.0 -27 -true 33 1969-12-31 16:00:09.924 33.0 cvLH6Eat2yFsyy7p -33 33 0 33.0 0.0 -33.0 2625.249 0.0 -33.0 0.0 -10.175 33.0 -0.7963636363636364 33.0 0.0 33 -NULL -18 1969-12-31 16:00:00.564 -18.0 NULL 18 -18 0 -18.0 -0.0 18.0 -1431.954 0.0 18.0 0.0 -10.175 -18.0 1.46 -18.0 0.0 -18 -NULL -51 1969-12-31 16:00:02.011 -51.0 NULL 51 -51 0 -51.0 -0.0 51.0 -4057.203 0.0 51.0 0.0 -10.175 -51.0 0.5152941176470588 -51.0 0.0 -51 -true -8 1969-12-31 16:00:02.109 -8.0 cvLH6Eat2yFsyy7p 8 -8 0 -8.0 -0.0 8.0 -636.424 0.0 8.0 0.0 -10.175 -8.0 3.285 -8.0 0.0 -8 -true -13 1969-12-31 16:00:15.223 -13.0 1cGVWH7n1QU 13 -13 0 -13.0 -0.0 13.0 -1034.1889999999999 0.0 13.0 0.0 -10.175 -13.0 2.0215384615384617 -13.0 0.0 -13 -true -52 1969-12-31 16:00:01.824 -52.0 821UdmGbkEf4j 52 -52 0 -52.0 -0.0 52.0 -4136.755999999999 0.0 52.0 0.0 -10.175 -52.0 0.5053846153846154 -52.0 0.0 -52 -true -28 1969-12-31 16:00:10.554 -28.0 821UdmGbkEf4j 28 -28 0 -28.0 -0.0 28.0 -2227.484 0.0 28.0 0.0 -10.175 -28.0 0.9385714285714286 -28.0 0.0 -28 -true 2 1969-12-31 16:00:06.546 2.0 821UdmGbkEf4j -2 2 0 2.0 0.0 -2.0 159.106 0.0 -2.0 0.0 -10.175 2.0 -13.14 2.0 0.0 2 -true 40 1969-12-31 16:00:02.842 40.0 1cGVWH7n1QU -40 40 0 40.0 0.0 -40.0 3182.12 0.0 -40.0 0.0 -10.175 40.0 -0.657 40.0 0.0 40 -NULL 55 1969-12-31 16:00:02.215 55.0 NULL -55 55 0 55.0 0.0 -55.0 4375.415 0.0 -55.0 0.0 -10.175 55.0 -0.47781818181818186 55.0 0.0 55 -NULL 22 1969-12-31 16:00:00.621 22.0 NULL -22 22 0 22.0 0.0 -22.0 1750.166 0.0 -22.0 0.0 -10.175 22.0 -1.1945454545454546 22.0 0.0 22 -NULL 58 1969-12-31 16:00:03.366 58.0 NULL -58 58 0 58.0 0.0 -58.0 4614.074 0.0 -58.0 0.0 -10.175 58.0 -0.4531034482758621 58.0 0.0 58 -true 0 1969-12-31 16:00:08.742 0.0 821UdmGbkEf4j 0 0 0 0.0 0.0 -0.0 0.0 0.0 -0.0 0.0 -10.175 0.0 NULL 0.0 NULL 0 +NULL -64 1969-12-31 16:00:00.199 -64.0 NULL 64 -64 0 -64.0 -0.0 64.0 -5091.392 0.0 64.0 0.0 -10.175 -64.0 0.410625 -64.0 0.0 -64 +NULL -64 1969-12-31 16:00:00.29 -64.0 NULL 64 -64 0 -64.0 -0.0 64.0 -5091.392 0.0 64.0 0.0 -10.175 -64.0 0.410625 -64.0 0.0 -64 +NULL -64 1969-12-31 16:00:01.785 -64.0 NULL 64 -64 0 -64.0 -0.0 64.0 -5091.392 0.0 64.0 0.0 -10.175 -64.0 0.410625 -64.0 0.0 -64 +NULL -64 1969-12-31 16:00:03.944 -64.0 NULL 64 -64 0 -64.0 -0.0 64.0 -5091.392 0.0 64.0 0.0 -10.175 -64.0 0.410625 -64.0 0.0 -64 +NULL -64 1969-12-31 16:00:05.997 -64.0 NULL 64 -64 0 -64.0 -0.0 64.0 -5091.392 0.0 64.0 0.0 -10.175 -64.0 0.410625 -64.0 0.0 -64 +NULL -64 1969-12-31 16:00:10.858 -64.0 NULL 64 -64 0 -64.0 -0.0 64.0 -5091.392 0.0 64.0 0.0 -10.175 -64.0 0.410625 -64.0 0.0 -64 +NULL -64 1969-12-31 16:00:11.912 -64.0 NULL 64 -64 0 -64.0 -0.0 64.0 -5091.392 0.0 64.0 0.0 -10.175 -64.0 0.410625 -64.0 0.0 -64 +NULL -64 1969-12-31 16:00:12.339 -64.0 NULL 64 -64 0 -64.0 -0.0 64.0 -5091.392 0.0 64.0 0.0 -10.175 -64.0 0.410625 -64.0 0.0 -64 +NULL -64 1969-12-31 16:00:13.274 -64.0 NULL 64 -64 0 -64.0 -0.0 64.0 -5091.392 0.0 64.0 0.0 -10.175 -64.0 0.410625 -64.0 0.0 -64 +NULL -63 1969-12-31 16:00:01.843 -63.0 NULL 63 -63 0 -63.0 -0.0 63.0 -5011.839 0.0 63.0 0.0 -10.175 -63.0 0.41714285714285715 -63.0 0.0 -63 +NULL -63 1969-12-31 16:00:03.552 -63.0 NULL 63 -63 0 -63.0 -0.0 63.0 -5011.839 0.0 63.0 0.0 -10.175 -63.0 0.41714285714285715 -63.0 0.0 -63 +NULL -63 1969-12-31 16:00:06.852 -63.0 NULL 63 -63 0 -63.0 -0.0 63.0 -5011.839 0.0 63.0 0.0 -10.175 -63.0 0.41714285714285715 -63.0 0.0 -63 +NULL -63 1969-12-31 16:00:07.375 -63.0 NULL 63 -63 0 -63.0 -0.0 63.0 -5011.839 0.0 63.0 0.0 -10.175 -63.0 0.41714285714285715 -63.0 0.0 -63 +NULL -63 1969-12-31 16:00:10.205 -63.0 NULL 63 -63 0 -63.0 -0.0 63.0 -5011.839 0.0 63.0 0.0 -10.175 -63.0 0.41714285714285715 -63.0 0.0 -63 +NULL -63 1969-12-31 16:00:11.946 -63.0 NULL 63 -63 0 -63.0 -0.0 63.0 -5011.839 0.0 63.0 0.0 -10.175 -63.0 0.41714285714285715 -63.0 0.0 -63 +NULL -63 1969-12-31 16:00:12.188 -63.0 NULL 63 -63 0 -63.0 -0.0 63.0 -5011.839 0.0 63.0 0.0 -10.175 -63.0 0.41714285714285715 -63.0 0.0 -63 +NULL -63 1969-12-31 16:00:15.436 -63.0 NULL 63 -63 0 -63.0 -0.0 63.0 -5011.839 0.0 63.0 0.0 -10.175 -63.0 0.41714285714285715 -63.0 0.0 -63 +NULL -62 1969-12-31 16:00:00.037 -62.0 NULL 62 -62 0 -62.0 -0.0 62.0 -4932.286 0.0 62.0 0.0 -10.175 -62.0 0.4238709677419355 -62.0 0.0 -62 +NULL -62 1969-12-31 16:00:01.22 -62.0 NULL 62 -62 0 -62.0 -0.0 62.0 -4932.286 0.0 62.0 0.0 -10.175 -62.0 0.4238709677419355 -62.0 0.0 -62 +NULL -62 1969-12-31 16:00:01.515 -62.0 NULL 62 -62 0 -62.0 -0.0 62.0 -4932.286 0.0 62.0 0.0 -10.175 -62.0 0.4238709677419355 -62.0 0.0 -62 +NULL -62 1969-12-31 16:00:01.734 -62.0 NULL 62 -62 0 -62.0 -0.0 62.0 -4932.286 0.0 62.0 0.0 -10.175 -62.0 0.4238709677419355 -62.0 0.0 -62 +NULL -62 1969-12-31 16:00:02.373 -62.0 NULL 62 -62 0 -62.0 -0.0 62.0 -4932.286 0.0 62.0 0.0 -10.175 -62.0 0.4238709677419355 -62.0 0.0 -62 +NULL -62 1969-12-31 16:00:03.85 -62.0 NULL 62 -62 0 -62.0 -0.0 62.0 -4932.286 0.0 62.0 0.0 -10.175 -62.0 0.4238709677419355 -62.0 0.0 -62 +NULL -62 1969-12-31 16:00:08.198 -62.0 NULL 62 -62 0 -62.0 -0.0 62.0 -4932.286 0.0 62.0 0.0 -10.175 -62.0 0.4238709677419355 -62.0 0.0 -62 +NULL -62 1969-12-31 16:00:09.025 -62.0 NULL 62 -62 0 -62.0 -0.0 62.0 -4932.286 0.0 62.0 0.0 -10.175 -62.0 0.4238709677419355 -62.0 0.0 -62 +NULL -62 1969-12-31 16:00:09.889 -62.0 NULL 62 -62 0 -62.0 -0.0 62.0 -4932.286 0.0 62.0 0.0 -10.175 -62.0 0.4238709677419355 -62.0 0.0 -62 +NULL -62 1969-12-31 16:00:10.069 -62.0 NULL 62 -62 0 -62.0 -0.0 62.0 -4932.286 0.0 62.0 0.0 -10.175 -62.0 0.4238709677419355 -62.0 0.0 -62 +NULL -62 1969-12-31 16:00:10.225 -62.0 NULL 62 -62 0 -62.0 -0.0 62.0 -4932.286 0.0 62.0 0.0 -10.175 -62.0 0.4238709677419355 -62.0 0.0 -62 +NULL -62 1969-12-31 16:00:10.485 -62.0 NULL 62 -62 0 -62.0 -0.0 62.0 -4932.286 0.0 62.0 0.0 -10.175 -62.0 0.4238709677419355 -62.0 0.0 -62 +NULL -62 1969-12-31 16:00:12.388 -62.0 NULL 62 -62 0 -62.0 -0.0 62.0 -4932.286 0.0 62.0 0.0 -10.175 -62.0 0.4238709677419355 -62.0 0.0 -62 +NULL -62 1969-12-31 16:00:12.591 -62.0 NULL 62 -62 0 -62.0 -0.0 62.0 -4932.286 0.0 62.0 0.0 -10.175 -62.0 0.4238709677419355 -62.0 0.0 -62 +NULL -62 1969-12-31 16:00:14.154 -62.0 NULL 62 -62 0 -62.0 -0.0 62.0 -4932.286 0.0 62.0 0.0 -10.175 -62.0 0.4238709677419355 -62.0 0.0 -62 +NULL -62 1969-12-31 16:00:14.247 -62.0 NULL 62 -62 0 -62.0 -0.0 62.0 -4932.286 0.0 62.0 0.0 -10.175 -62.0 0.4238709677419355 -62.0 0.0 -62 +NULL -62 1969-12-31 16:00:14.517 -62.0 NULL 62 -62 0 -62.0 -0.0 62.0 -4932.286 0.0 62.0 0.0 -10.175 -62.0 0.4238709677419355 -62.0 0.0 -62 +NULL -62 1969-12-31 16:00:14.965 -62.0 NULL 62 -62 0 -62.0 -0.0 62.0 -4932.286 0.0 62.0 0.0 -10.175 -62.0 0.4238709677419355 -62.0 0.0 -62 +NULL -61 1969-12-31 16:00:00.142 -61.0 NULL 61 -61 0 -61.0 -0.0 61.0 -4852.733 0.0 61.0 0.0 -10.175 -61.0 0.4308196721311476 -61.0 0.0 -61 +NULL -61 1969-12-31 16:00:02.698 -61.0 NULL 61 -61 0 -61.0 -0.0 61.0 -4852.733 0.0 61.0 0.0 -10.175 -61.0 0.4308196721311476 -61.0 0.0 -61 +NULL -61 1969-12-31 16:00:03.049 -61.0 NULL 61 -61 0 -61.0 -0.0 61.0 -4852.733 0.0 61.0 0.0 -10.175 -61.0 0.4308196721311476 -61.0 0.0 -61 +NULL -61 1969-12-31 16:00:04.165 -61.0 NULL 61 -61 0 -61.0 -0.0 61.0 -4852.733 0.0 61.0 0.0 -10.175 -61.0 0.4308196721311476 -61.0 0.0 -61 +NULL -61 1969-12-31 16:00:04.977 -61.0 NULL 61 -61 0 -61.0 -0.0 61.0 -4852.733 0.0 61.0 0.0 -10.175 -61.0 0.4308196721311476 -61.0 0.0 -61 diff --git a/ql/src/test/results/clientpositive/spark/vectorization_short_regress.q.out b/ql/src/test/results/clientpositive/spark/vectorization_short_regress.q.out index d02c96e..269c4e3 100644 --- a/ql/src/test/results/clientpositive/spark/vectorization_short_regress.q.out +++ b/ql/src/test/results/clientpositive/spark/vectorization_short_regress.q.out @@ -1866,7 +1866,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 1) + Reducer 2 <- Map 1 (GROUP, 3) Reducer 3 <- Reducer 2 (SORT, 1) #### A masked pattern was here #### Vertices: @@ -2074,7 +2074,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 1) + Reducer 2 <- Map 1 (GROUP, 3) Reducer 3 <- Reducer 2 (SORT, 1) #### A masked pattern was here #### Vertices: @@ -2329,7 +2329,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 1) + Reducer 2 <- Map 1 (GROUP, 3) Reducer 3 <- Reducer 2 (SORT, 1) #### A masked pattern was here #### Vertices: @@ -2662,7 +2662,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 1) + Reducer 2 <- Map 1 (GROUP, 3) Reducer 3 <- Reducer 2 (SORT, 1) #### A masked pattern was here #### Vertices: diff --git a/ql/src/test/results/clientpositive/spark/vectorized_ptf.q.out b/ql/src/test/results/clientpositive/spark/vectorized_ptf.q.out index 1028345..5fe0020 100644 --- a/ql/src/test/results/clientpositive/spark/vectorized_ptf.q.out +++ b/ql/src/test/results/clientpositive/spark/vectorized_ptf.q.out @@ -228,8 +228,8 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1) - Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3) + Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -588,8 +588,8 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1) - Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3) + Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -839,7 +839,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -1095,8 +1095,8 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1) - Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3) + Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -1383,8 +1383,8 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1) - Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3) + Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -1680,9 +1680,9 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1) - Reducer 3 <- Reducer 2 (GROUP, 1) - Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3) + Reducer 3 <- Reducer 2 (GROUP, 3) + Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -2020,7 +2020,7 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 3 <- Map 2 (PARTITION-LEVEL SORT, 1) + Reducer 3 <- Map 2 (PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 2 @@ -2263,7 +2263,7 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 3 <- Map 2 (PARTITION-LEVEL SORT, 1) + Reducer 3 <- Map 2 (PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 2 @@ -2589,8 +2589,8 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1) - Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3) + Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -2866,8 +2866,8 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1) - Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3) + Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -3147,8 +3147,8 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1) - Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3) + Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -3435,9 +3435,9 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1) - Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 1) - Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3) + Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3) + Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -3762,8 +3762,8 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1) - Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3) + Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -4222,8 +4222,8 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 3 <- Map 2 (PARTITION-LEVEL SORT, 1) - Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 1) + Reducer 3 <- Map 2 (PARTITION-LEVEL SORT, 3) + Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 2 @@ -4478,8 +4478,8 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1) - Reducer 3 <- Reducer 2 (GROUP, 1) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3) + Reducer 3 <- Reducer 2 (GROUP, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -4749,9 +4749,9 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (GROUP, 1) - Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 1) - Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (GROUP, 3) + Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3) + Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -5222,11 +5222,11 @@ STAGE PLANS: Stage: Stage-2 Spark Edges: - Reducer 6 <- Map 1 (PARTITION-LEVEL SORT, 1) - Reducer 7 <- Map 1 (PARTITION-LEVEL SORT, 1) - Reducer 3 <- Reducer 6 (PARTITION-LEVEL SORT, 1) - Reducer 5 <- Reducer 7 (PARTITION-LEVEL SORT, 1) - Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 1) + Reducer 6 <- Map 1 (PARTITION-LEVEL SORT, 3) + Reducer 7 <- Map 1 (PARTITION-LEVEL SORT, 3) + Reducer 3 <- Reducer 6 (PARTITION-LEVEL SORT, 3) + Reducer 5 <- Reducer 7 (PARTITION-LEVEL SORT, 3) + Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -5736,9 +5736,9 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1) - Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 1) - Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3) + Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3) + Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -6092,10 +6092,10 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1) - Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 1) - Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 1) - Reducer 5 <- Reducer 4 (PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3) + Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3) + Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 3) + Reducer 5 <- Reducer 4 (PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -6446,9 +6446,9 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1) - Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 1) - Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3) + Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3) + Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -6795,10 +6795,10 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1) - Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 1) - Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 1) - Reducer 5 <- Reducer 4 (PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3) + Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3) + Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 3) + Reducer 5 <- Reducer 4 (PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -7194,9 +7194,9 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1) - Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 1) - Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3) + Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3) + Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 @@ -7543,9 +7543,9 @@ STAGE PLANS: Stage: Stage-1 Spark Edges: - Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1) - Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 1) - Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 1) + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3) + Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 3) + Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 3) #### A masked pattern was here #### Vertices: Map 1 diff --git a/spark-client/src/main/java/org/apache/hive/spark/client/ClientUtils.java b/spark-client/src/main/java/org/apache/hive/spark/client/ClientUtils.java index 47c13d1..98f5ad2 100644 --- a/spark-client/src/main/java/org/apache/hive/spark/client/ClientUtils.java +++ b/spark-client/src/main/java/org/apache/hive/spark/client/ClientUtils.java @@ -64,7 +64,7 @@ static ActorSystemInfo createActorSystem(Map conf) throws IOExce int akkaHeartBeatPauses = toInt(conf.get("spark.akka.heartbeat.pauses"), 600); double akkaFailureDetector = toDouble(conf.get("spark.akka.failure-detector.threshold"), 300.0); - int akkaHeartBeatInterval = toInt(conf.get("spark.akka.heartbeat.interval"), 1000); + int akkaHeartBeatInterval = toInt(conf.get("spark.akka.heartbeat.interval"), 100); // Disabled due to chill-akka depending on kryo 2.21, which is incompatible with 2.22 // due to packaging changes (relocated org.objenesis classes). diff --git a/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientImpl.java b/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientImpl.java index 0aea4a2..903169d 100644 --- a/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientImpl.java +++ b/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientImpl.java @@ -17,10 +17,12 @@ package org.apache.hive.spark.client; +import java.io.BufferedReader; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; +import java.io.InputStreamReader; import java.io.OutputStream; import java.io.OutputStreamWriter; import java.io.Serializable; @@ -192,8 +194,12 @@ public void run() { // If a Spark installation is provided, use the spark-submit script. Otherwise, call the // SparkSubmit class directly, which has some caveats (like having to provide a proper // version of Guava on the classpath depending on the deploy mode). - if (conf.get("spark.home") != null) { - argv.add(new File(conf.get("spark.home"), "bin/spark-submit").getAbsolutePath()); + String sparkHome = conf.get("spark.home"); + if (sparkHome == null) { + sparkHome = System.getProperty("spark.home"); + } + if (sparkHome != null) { + argv.add(new File(sparkHome, "bin/spark-submit").getAbsolutePath()); } else { LOG.info("No spark.home provided, calling SparkSubmit directly."); argv.add(new File(System.getProperty("java.home"), "bin/java").getAbsolutePath()); @@ -248,12 +254,11 @@ public void run() { LOG.debug("Running client driver with argv: {}", Joiner.on(" ").join(argv)); ProcessBuilder pb = new ProcessBuilder(argv.toArray(new String[argv.size()])); - pb.environment().clear(); final Process child = pb.start(); int childId = childIdGenerator.incrementAndGet(); - redirect("stdout-redir-" + childId, child.getInputStream(), System.out); - redirect("stderr-redir-" + childId, child.getErrorStream(), System.err); + redirect("stdout-redir-" + childId, child.getInputStream()); + redirect("stderr-redir-" + childId, child.getErrorStream()); runnable = new Runnable() { @Override @@ -277,8 +282,8 @@ public void run() { return thread; } - private void redirect(String name, InputStream in, OutputStream out) { - Thread thread = new Thread(new Redirector(in, out)); + private void redirect(String name, InputStream in) { + Thread thread = new Thread(new Redirector(in)); thread.setName(name); thread.setDaemon(true); thread.start(); @@ -345,23 +350,18 @@ public void onReceive(Object message) throws Exception { private class Redirector implements Runnable { - private final InputStream in; - private final OutputStream out; + private final BufferedReader in; - Redirector(InputStream in, OutputStream out) { - this.in = in; - this.out = out; + Redirector(InputStream in) { + this.in = new BufferedReader(new InputStreamReader(in)); } @Override public void run() { try { - byte[] buf = new byte[1024]; - int len = in.read(buf); - while (len != -1) { - out.write(buf, 0, len); - out.flush(); - len = in.read(buf); + String line = null; + while ((line = in.readLine()) != null) { + LOG.info(line); } } catch (Exception e) { LOG.warn("Error in redirector thread.", e);