diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java index d7d8b6fee1..6efc6d3d06 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java @@ -2748,7 +2748,11 @@ private boolean validateGroupByOperator(GroupByOperator op, boolean isReduce, return false; } - if (!validateAggregationDescs(desc.getAggregators(), desc.getMode(), hasKeys)) { + //TODO: isGroupingSetsPresent() is returning false, even though + // ListGroupingSets is present. Need to check if there is hidden bug. + boolean isGroupingSetsPresent = (desc.getListGroupingSets() != null && !desc.getListGroupingSets().isEmpty()); + if (!validateAggregationDescs(desc.getAggregators(), desc.getMode(), + isGroupingSetsPresent, hasKeys)) { return false; } @@ -3002,10 +3006,12 @@ private boolean validateExprNodeDesc(List descs, } private boolean validateAggregationDescs(List descs, - GroupByDesc.Mode groupByMode, boolean hasKeys) { + GroupByDesc.Mode groupByMode, boolean isGroupingSetsPresent, + boolean hasKeys) { for (AggregationDesc d : descs) { - if (!validateAggregationDesc(d, groupByMode, hasKeys)) { + if (!validateAggregationDesc(d, groupByMode, isGroupingSetsPresent, + hasKeys)) { return false; } } @@ -3162,7 +3168,7 @@ private boolean validateGenericUdf(ExprNodeGenericFuncDesc genericUDFExpr) { } private boolean validateAggregationDesc(AggregationDesc aggDesc, GroupByDesc.Mode groupByMode, - boolean hasKeys) { + boolean isGroupingSetsPresent, boolean hasKeys) { String udfName = aggDesc.getGenericUDAFName().toLowerCase(); if (!supportedAggregationUdfs.contains(udfName)) { @@ -3171,11 +3177,16 @@ private boolean validateAggregationDesc(AggregationDesc aggDesc, GroupByDesc.Mod } // The planner seems to pull this one out. - if (aggDesc.getDistinct()) { + if (groupByMode != GroupByDesc.Mode.HASH && aggDesc.getDistinct()) { setExpressionIssue("Aggregation Function", "DISTINCT not supported"); return false; } + if (isGroupingSetsPresent && aggDesc.getDistinct()) { + setExpressionIssue("Aggregation Function", "DISTINCT with Groupingsets not supported"); + return false; + } + List parameters = aggDesc.getParameters(); diff --git a/ql/src/test/queries/clientpositive/vectorized_distinct_gby.q b/ql/src/test/queries/clientpositive/vectorized_distinct_gby.q index a64a60718f..c231cd2142 100644 --- a/ql/src/test/queries/clientpositive/vectorized_distinct_gby.q +++ b/ql/src/test/queries/clientpositive/vectorized_distinct_gby.q @@ -18,3 +18,14 @@ select sum(distinct a), count(distinct a) from dtest; explain vectorization detail select sum(distinct cint), count(distinct cint), avg(distinct cint), std(distinct cint) from alltypesorc; select sum(distinct cint), count(distinct cint), avg(distinct cint), std(distinct cint) from alltypesorc; + +explain vectorization detail +select ctinyint, count(distinct cint), sum(( CASE WHEN ( ( cstring1 LIKE'test%1' ) + OR ( cstring1 LIKE 'test%2' ) ) THEN 1 ELSE 0 END )) AS s, +max(( CASE WHEN ( ( cstring1 LIKE 'test%3' ) OR ( cstring1 LIKE '%test%5' ) ) +THEN cstring1 ELSE 'XXXXX' END )) AS maxVal from alltypesorc group by ctinyint; + +select ctinyint, count(distinct cint), sum(( CASE WHEN ( ( cstring1 LIKE +'test%1' ) OR ( cstring1 LIKE 'test%2' ) ) THEN 1 ELSE 0 END )) AS s, +max(( CASE WHEN ( ( cstring1 LIKE 'test%3' ) OR ( cstring1 LIKE '%test%5' ) ) +THEN cstring1 ELSE 'XXXXX' END )) AS maxVal from alltypesorc group by ctinyint; diff --git a/ql/src/test/results/clientpositive/llap/vector_count.q.out b/ql/src/test/results/clientpositive/llap/vector_count.q.out index 65f21b5699..1b196ce667 100644 --- a/ql/src/test/results/clientpositive/llap/vector_count.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_count.q.out @@ -72,12 +72,26 @@ STAGE PLANS: TableScan alias: abcd Statistics: Num rows: 7 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE + TableScan Vectorization: + native: true Select Operator expressions: a (type: int), b (type: int), c (type: int), d (type: int) outputColumnNames: a, b, c, d + Select Vectorization: + className: VectorSelectOperator + native: true + projectedOutputColumnNums: [0, 1, 2, 3] Statistics: Num rows: 7 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count(DISTINCT b), count(DISTINCT c), sum(d) + Group By Vectorization: + aggregators: VectorUDAFCount(col 1:int) -> bigint, VectorUDAFCount(col 2:int) -> bigint, VectorUDAFSumLong(col 3:int) -> bigint + className: VectorGroupByOperator + groupByMode: HASH + keyExpressions: col 0:int, col 1:int, col 2:int + native: false + vectorProcessingMode: HASH + projectedOutputColumnNums: [0, 1, 2] keys: a (type: int), b (type: int), c (type: int) minReductionHashAggr: 0.14285713 mode: hash @@ -88,16 +102,24 @@ STAGE PLANS: null sort order: zzz sort order: +++ Map-reduce partition columns: _col0 (type: int) + Reduce Sink Vectorization: + className: VectorReduceSinkOperator + native: false + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + nativeConditionsNotMet: No DISTINCT columns IS false Statistics: Num rows: 3 Data size: 108 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col5 (type: bigint) - Execution mode: llap + Execution mode: vectorized, llap LLAP IO: all inputs Map Vectorization: enabled: true enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - notVectorizedReason: Aggregation Function expression for GROUPBY operator: DISTINCT not supported - vectorized: false + allNative: false + usesVectorUDFAdaptor: false + vectorized: true Reducer 2 Execution mode: llap Reduce Vectorization: @@ -190,7 +212,7 @@ STAGE PLANS: enabled: true enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - notVectorizedReason: Aggregation Function expression for GROUPBY operator: DISTINCT not supported + notVectorizedReason: GROUPBY operator: Aggregations with > 1 parameter are not supported count([Column[a], Column[b]]) vectorized: false Reducer 2 Execution mode: llap diff --git a/ql/src/test/results/clientpositive/llap/vector_groupby_cube1.q.out b/ql/src/test/results/clientpositive/llap/vector_groupby_cube1.q.out index c30f7ff2c1..96c83be6b2 100644 --- a/ql/src/test/results/clientpositive/llap/vector_groupby_cube1.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_groupby_cube1.q.out @@ -548,7 +548,7 @@ STAGE PLANS: enabled: true enabledConditionsMet: hive.vectorized.use.vector.serde.deserialize IS true inputFileFormats: org.apache.hadoop.mapred.TextInputFormat - notVectorizedReason: Aggregation Function expression for GROUPBY operator: DISTINCT not supported + notVectorizedReason: Aggregation Function expression for GROUPBY operator: DISTINCT with Groupingsets not supported vectorized: false Reducer 2 Execution mode: llap @@ -867,7 +867,7 @@ STAGE PLANS: enabled: true enabledConditionsMet: hive.vectorized.use.vector.serde.deserialize IS true inputFileFormats: org.apache.hadoop.mapred.TextInputFormat - notVectorizedReason: Aggregation Function expression for GROUPBY operator: DISTINCT not supported + notVectorizedReason: Aggregation Function expression for GROUPBY operator: DISTINCT with Groupingsets not supported vectorized: false Reducer 2 Execution mode: llap diff --git a/ql/src/test/results/clientpositive/llap/vector_groupby_rollup1.q.out b/ql/src/test/results/clientpositive/llap/vector_groupby_rollup1.q.out index ce27c9d147..e6ae542cbe 100644 --- a/ql/src/test/results/clientpositive/llap/vector_groupby_rollup1.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_groupby_rollup1.q.out @@ -243,7 +243,7 @@ STAGE PLANS: enabled: true enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - notVectorizedReason: Aggregation Function expression for GROUPBY operator: DISTINCT not supported + notVectorizedReason: Aggregation Function expression for GROUPBY operator: DISTINCT with Groupingsets not supported vectorized: false Reducer 2 Execution mode: llap @@ -556,7 +556,7 @@ STAGE PLANS: enabled: true enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - notVectorizedReason: Aggregation Function expression for GROUPBY operator: DISTINCT not supported + notVectorizedReason: Aggregation Function expression for GROUPBY operator: DISTINCT with Groupingsets not supported vectorized: false Reducer 2 Execution mode: llap diff --git a/ql/src/test/results/clientpositive/llap/vectorized_distinct_gby.q.out b/ql/src/test/results/clientpositive/llap/vectorized_distinct_gby.q.out index 60cd2e3268..ca71b6e8fe 100644 --- a/ql/src/test/results/clientpositive/llap/vectorized_distinct_gby.q.out +++ b/ql/src/test/results/clientpositive/llap/vectorized_distinct_gby.q.out @@ -371,3 +371,266 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@alltypesorc #### A masked pattern was here #### -3482841611 6082 -572647.4204209142 6.153814687328988E8 +PREHOOK: query: explain vectorization detail +select ctinyint, count(distinct cint), sum(( CASE WHEN ( ( cstring1 LIKE'test%1' ) + OR ( cstring1 LIKE 'test%2' ) ) THEN 1 ELSE 0 END )) AS s, +max(( CASE WHEN ( ( cstring1 LIKE 'test%3' ) OR ( cstring1 LIKE '%test%5' ) ) +THEN cstring1 ELSE 'XXXXX' END )) AS maxVal from alltypesorc group by ctinyint +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypesorc +#### A masked pattern was here #### +POSTHOOK: query: explain vectorization detail +select ctinyint, count(distinct cint), sum(( CASE WHEN ( ( cstring1 LIKE'test%1' ) + OR ( cstring1 LIKE 'test%2' ) ) THEN 1 ELSE 0 END )) AS s, +max(( CASE WHEN ( ( cstring1 LIKE 'test%3' ) OR ( cstring1 LIKE '%test%5' ) ) +THEN cstring1 ELSE 'XXXXX' END )) AS maxVal from alltypesorc group by ctinyint +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alltypesorc +#### A masked pattern was here #### +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: alltypesorc + Statistics: Num rows: 12288 Data size: 935842 Basic stats: COMPLETE Column stats: COMPLETE + TableScan Vectorization: + native: true + vectorizationSchemaColumns: [0:ctinyint:tinyint, 1:csmallint:smallint, 2:cint:int, 3:cbigint:bigint, 4:cfloat:float, 5:cdouble:double, 6:cstring1:string, 7:cstring2:string, 8:ctimestamp1:timestamp, 9:ctimestamp2:timestamp, 10:cboolean1:boolean, 11:cboolean2:boolean, 12:ROW__ID:struct] + Select Operator + expressions: ctinyint (type: tinyint), cint (type: int), CASE WHEN (((cstring1 like 'test%1') or (cstring1 like 'test%2'))) THEN (1) ELSE (0) END (type: int), CASE WHEN (((cstring1 like 'test%3') or (cstring1 like '%test%5'))) THEN (cstring1) ELSE ('XXXXX') END (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Select Vectorization: + className: VectorSelectOperator + native: true + projectedOutputColumnNums: [0, 2, 16, 20] + selectExpressions: IfExprLongScalarLongScalar(col 15:boolean, val 1, val 0)(children: ColOrCol(col 13:boolean, col 14:boolean)(children: SelectStringColLikeStringScalar(col 6:string) -> 13:boolean, SelectStringColLikeStringScalar(col 6:string) -> 14:boolean) -> 15:boolean) -> 16:int, IfExprStringGroupColumnStringScalar(col 19:boolean, col 6:string, val XXXXX)(children: ColOrCol(col 17:boolean, col 18:boolean)(children: SelectStringColLikeStringScalar(col 6:string) -> 17:boolean, SelectStringColLikeStringScalar(col 6:string) -> 18:boolean) -> 19:boolean) -> 20:string + Statistics: Num rows: 12288 Data size: 935842 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count(DISTINCT _col1), sum(_col2), max(_col3) + Group By Vectorization: + aggregators: VectorUDAFCount(col 2:int) -> bigint, VectorUDAFSumLong(col 16:int) -> bigint, VectorUDAFMaxString(col 20:string) -> string + className: VectorGroupByOperator + groupByMode: HASH + keyExpressions: col 0:tinyint, col 2:int + native: false + vectorProcessingMode: HASH + projectedOutputColumnNums: [0, 1, 2] + keys: _col0 (type: tinyint), _col1 (type: int) + minReductionHashAggr: 0.0 + mode: hash + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 12288 Data size: 2530992 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col0 (type: tinyint), _col1 (type: int) + null sort order: zz + sort order: ++ + Map-reduce partition columns: _col0 (type: tinyint) + Reduce Sink Vectorization: + className: VectorReduceSinkOperator + native: false + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + nativeConditionsNotMet: No DISTINCT columns IS false + Statistics: Num rows: 12288 Data size: 2530992 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col3 (type: bigint), _col4 (type: string) + Execution mode: vectorized, llap + LLAP IO: all inputs + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] + inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + allNative: false + usesVectorUDFAdaptor: false + vectorized: true + rowBatchContext: + dataColumnCount: 12 + includeColumns: [0, 2, 6] + dataColumns: ctinyint:tinyint, csmallint:smallint, cint:int, cbigint:bigint, cfloat:float, cdouble:double, cstring1:string, cstring2:string, ctimestamp1:timestamp, ctimestamp2:timestamp, cboolean1:boolean, cboolean2:boolean + partitionColumnCount: 0 + scratchColumnTypeNames: [bigint, bigint, bigint, bigint, bigint, bigint, bigint, string] + Reducer 2 + Execution mode: llap + Reduce Vectorization: + enabled: true + enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true + notVectorizedReason: GROUPBY operator: DISTINCT not supported + vectorized: false + Reduce Operator Tree: + Group By Operator + aggregations: count(DISTINCT KEY._col1:0._col0), sum(VALUE._col1), max(VALUE._col2) + keys: KEY._col0 (type: tinyint) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 131 Data size: 26596 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 131 Data size: 26596 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select ctinyint, count(distinct cint), sum(( CASE WHEN ( ( cstring1 LIKE +'test%1' ) OR ( cstring1 LIKE 'test%2' ) ) THEN 1 ELSE 0 END )) AS s, +max(( CASE WHEN ( ( cstring1 LIKE 'test%3' ) OR ( cstring1 LIKE '%test%5' ) ) +THEN cstring1 ELSE 'XXXXX' END )) AS maxVal from alltypesorc group by ctinyint +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypesorc +#### A masked pattern was here #### +POSTHOOK: query: select ctinyint, count(distinct cint), sum(( CASE WHEN ( ( cstring1 LIKE +'test%1' ) OR ( cstring1 LIKE 'test%2' ) ) THEN 1 ELSE 0 END )) AS s, +max(( CASE WHEN ( ( cstring1 LIKE 'test%3' ) OR ( cstring1 LIKE '%test%5' ) ) +THEN cstring1 ELSE 'XXXXX' END )) AS maxVal from alltypesorc group by ctinyint +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alltypesorc +#### A masked pattern was here #### +-64 3 0 XXXXX +-63 3 0 XXXXX +-62 3 0 XXXXX +-61 3 0 XXXXX +-60 3 0 XXXXX +-59 3 0 XXXXX +-58 3 0 XXXXX +-57 3 0 XXXXX +-56 3 0 XXXXX +-55 3 0 XXXXX +-54 3 0 XXXXX +-53 3 0 XXXXX +-52 3 0 XXXXX +-51 1009 0 XXXXX +-50 3 0 XXXXX +-49 3 0 XXXXX +-48 3 0 XXXXX +-47 3 0 XXXXX +-46 3 0 XXXXX +-45 3 0 XXXXX +-44 3 0 XXXXX +-43 3 0 XXXXX +-42 3 0 XXXXX +-41 3 0 XXXXX +-40 3 0 XXXXX +-39 3 0 XXXXX +-38 3 0 XXXXX +-37 3 0 XXXXX +-36 3 0 XXXXX +-35 3 0 XXXXX +-34 3 0 XXXXX +-33 3 0 XXXXX +-32 3 0 XXXXX +-31 3 0 XXXXX +-30 3 0 XXXXX +-29 3 0 XXXXX +-28 3 0 XXXXX +-27 3 0 XXXXX +-26 3 0 XXXXX +-25 3 0 XXXXX +-24 3 0 XXXXX +-23 3 0 XXXXX +-22 3 0 XXXXX +-21 3 0 XXXXX +-20 3 0 XXXXX +-19 3 0 XXXXX +-18 3 0 XXXXX +-17 3 0 XXXXX +-16 3 0 XXXXX +-15 3 0 XXXXX +-14 3 0 XXXXX +-13 3 0 XXXXX +-12 3 0 XXXXX +-11 3 0 XXXXX +-10 3 0 XXXXX +-9 3 0 XXXXX +-8 3 0 XXXXX +-7 3 0 XXXXX +-6 3 0 XXXXX +-5 3 0 XXXXX +-4 3 0 XXXXX +-3 3 0 XXXXX +-2 3 0 XXXXX +-1 3 0 XXXXX +0 3 0 XXXXX +1 3 0 XXXXX +2 3 0 XXXXX +3 3 0 XXXXX +4 3 0 XXXXX +5 3 0 XXXXX +6 3 0 XXXXX +7 3 0 XXXXX +8 1011 0 XXXXX +9 3 0 XXXXX +10 3 0 XXXXX +11 1011 0 XXXXX +12 3 0 XXXXX +13 3 0 XXXXX +14 3 0 XXXXX +15 3 0 XXXXX +16 3 0 XXXXX +17 3 0 XXXXX +18 3 0 XXXXX +19 3 0 XXXXX +20 3 0 XXXXX +21 3 0 XXXXX +22 3 0 XXXXX +23 3 0 XXXXX +24 3 0 XXXXX +25 3 0 XXXXX +26 3 0 XXXXX +27 3 0 XXXXX +28 3 0 XXXXX +29 3 0 XXXXX +30 3 0 XXXXX +31 3 0 XXXXX +32 3 0 XXXXX +33 3 0 XXXXX +34 3 0 XXXXX +35 3 0 XXXXX +36 3 0 XXXXX +37 3 0 XXXXX +38 3 0 XXXXX +39 3 0 XXXXX +40 3 0 XXXXX +41 3 0 XXXXX +42 3 0 XXXXX +43 3 0 XXXXX +44 3 0 XXXXX +45 3 0 XXXXX +46 3 0 XXXXX +47 3 0 XXXXX +48 3 0 XXXXX +49 3 0 XXXXX +50 3 0 XXXXX +51 3 0 XXXXX +52 3 0 XXXXX +53 3 0 XXXXX +54 3 0 XXXXX +55 3 0 XXXXX +56 3 0 XXXXX +57 3 0 XXXXX +58 3 0 XXXXX +59 3 0 XXXXX +60 3 0 XXXXX +61 3 0 XXXXX +62 3 0 XXXXX +NULL 3066 0 XXXXX diff --git a/ql/src/test/results/clientpositive/spark/count.q.out b/ql/src/test/results/clientpositive/spark/count.q.out index dd2383518b..4688f6ede4 100644 --- a/ql/src/test/results/clientpositive/spark/count.q.out +++ b/ql/src/test/results/clientpositive/spark/count.q.out @@ -71,6 +71,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 1 Data size: 780 Basic stats: COMPLETE Column stats: NONE value expressions: _col5 (type: bigint) + Execution mode: vectorized Reducer 2 Reduce Operator Tree: Group By Operator diff --git a/ql/src/test/results/clientpositive/spark/groupby8_map_skew.q.out b/ql/src/test/results/clientpositive/spark/groupby8_map_skew.q.out index b491458d5b..d599f948f5 100644 --- a/ql/src/test/results/clientpositive/spark/groupby8_map_skew.q.out +++ b/ql/src/test/results/clientpositive/spark/groupby8_map_skew.q.out @@ -68,6 +68,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Execution mode: vectorized Reducer 2 Reduce Operator Tree: Group By Operator diff --git a/ql/src/test/results/clientpositive/spark/groupby9.q.out b/ql/src/test/results/clientpositive/spark/groupby9.q.out index 1f16390483..6a9c997edf 100644 --- a/ql/src/test/results/clientpositive/spark/groupby9.q.out +++ b/ql/src/test/results/clientpositive/spark/groupby9.q.out @@ -67,6 +67,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Execution mode: vectorized Map 5 Map Operator Tree: TableScan @@ -89,6 +90,7 @@ STAGE PLANS: sort order: +++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Execution mode: vectorized Reducer 2 Reduce Operator Tree: Group By Operator @@ -864,6 +866,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Execution mode: vectorized Map 5 Map Operator Tree: TableScan @@ -886,6 +889,7 @@ STAGE PLANS: sort order: +++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Execution mode: vectorized Reducer 2 Reduce Operator Tree: Group By Operator @@ -1661,6 +1665,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Execution mode: vectorized Map 5 Map Operator Tree: TableScan @@ -1683,6 +1688,7 @@ STAGE PLANS: sort order: +++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Execution mode: vectorized Reducer 2 Reduce Operator Tree: Group By Operator @@ -3261,6 +3267,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Execution mode: vectorized Map 5 Map Operator Tree: TableScan @@ -3283,6 +3290,7 @@ STAGE PLANS: sort order: +++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Execution mode: vectorized Reducer 2 Reduce Operator Tree: Group By Operator diff --git a/ql/src/test/results/clientpositive/spark/groupby_multi_insert_common_distinct.q.out b/ql/src/test/results/clientpositive/spark/groupby_multi_insert_common_distinct.q.out index 58b1359bb3..50e4345e5a 100644 --- a/ql/src/test/results/clientpositive/spark/groupby_multi_insert_common_distinct.q.out +++ b/ql/src/test/results/clientpositive/spark/groupby_multi_insert_common_distinct.q.out @@ -67,6 +67,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Execution mode: vectorized Map 5 Map Operator Tree: TableScan @@ -89,6 +90,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: double) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Execution mode: vectorized Reducer 2 Reduce Operator Tree: Group By Operator diff --git a/ql/src/test/results/clientpositive/spark/groupby_position.q.out b/ql/src/test/results/clientpositive/spark/groupby_position.q.out index 8749fd5be8..c5b5771ca0 100644 --- a/ql/src/test/results/clientpositive/spark/groupby_position.q.out +++ b/ql/src/test/results/clientpositive/spark/groupby_position.q.out @@ -66,6 +66,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + Execution mode: vectorized Map 5 Map Operator Tree: TableScan @@ -87,6 +88,7 @@ STAGE PLANS: sort order: +++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + Execution mode: vectorized Reducer 2 Reduce Operator Tree: Group By Operator @@ -269,6 +271,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + Execution mode: vectorized Map 5 Map Operator Tree: TableScan @@ -290,6 +293,7 @@ STAGE PLANS: sort order: +++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + Execution mode: vectorized Reducer 2 Reduce Operator Tree: Group By Operator diff --git a/ql/src/test/results/clientpositive/spark/multi_insert_gby3.q.out b/ql/src/test/results/clientpositive/spark/multi_insert_gby3.q.out index 04b6d25fe6..1af3f15242 100644 --- a/ql/src/test/results/clientpositive/spark/multi_insert_gby3.q.out +++ b/ql/src/test/results/clientpositive/spark/multi_insert_gby3.q.out @@ -83,6 +83,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Execution mode: vectorized Map 5 Map Operator Tree: TableScan @@ -262,6 +263,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Execution mode: vectorized Reducer 2 Execution mode: vectorized Reduce Operator Tree: @@ -1703,6 +1705,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Execution mode: vectorized Map 5 Map Operator Tree: TableScan @@ -1725,6 +1728,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Execution mode: vectorized Reducer 2 Reduce Operator Tree: Group By Operator @@ -1864,6 +1868,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Execution mode: vectorized Map 6 Map Operator Tree: TableScan @@ -1910,6 +1915,7 @@ STAGE PLANS: sort order: ++++ Map-reduce partition columns: _col0 (type: string), _col1 (type: double), _col2 (type: string) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Execution mode: vectorized Reducer 2 Reduce Operator Tree: Group By Operator diff --git a/ql/src/test/results/clientpositive/spark/nullgroup4.q.out b/ql/src/test/results/clientpositive/spark/nullgroup4.q.out index 80a55cfb57..15ee622471 100644 --- a/ql/src/test/results/clientpositive/spark/nullgroup4.q.out +++ b/ql/src/test/results/clientpositive/spark/nullgroup4.q.out @@ -47,6 +47,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) + Execution mode: vectorized Reducer 2 Reduce Operator Tree: Group By Operator diff --git a/ql/src/test/results/clientpositive/spark/nullgroup4_multi_distinct.q.out b/ql/src/test/results/clientpositive/spark/nullgroup4_multi_distinct.q.out index 875bef0b61..aca5200db0 100644 --- a/ql/src/test/results/clientpositive/spark/nullgroup4_multi_distinct.q.out +++ b/ql/src/test/results/clientpositive/spark/nullgroup4_multi_distinct.q.out @@ -45,6 +45,7 @@ STAGE PLANS: sort order: ++ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE value expressions: _col2 (type: bigint) + Execution mode: vectorized Reducer 2 Reduce Operator Tree: Group By Operator diff --git a/ql/src/test/results/clientpositive/spark/union17.q.out b/ql/src/test/results/clientpositive/spark/union17.q.out index c645207eb0..1157a014d0 100644 --- a/ql/src/test/results/clientpositive/spark/union17.q.out +++ b/ql/src/test/results/clientpositive/spark/union17.q.out @@ -92,6 +92,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1 Data size: 280 Basic stats: COMPLETE Column stats: PARTIAL + Execution mode: vectorized Map 7 Map Operator Tree: TableScan @@ -114,6 +115,7 @@ STAGE PLANS: sort order: +++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 1 Data size: 464 Basic stats: COMPLETE Column stats: PARTIAL + Execution mode: vectorized Reducer 3 Reduce Operator Tree: Group By Operator @@ -155,6 +157,7 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.dest2_n17 Reducer 8 + Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) @@ -179,6 +182,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 1 Data size: 280 Basic stats: COMPLETE Column stats: PARTIAL Reducer 9 + Execution mode: vectorized Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0)