diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java index 125bc65..57f7c01 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java @@ -2093,6 +2093,8 @@ private VectorExpression getGenericUdfVectorExpression(GenericUDF udf, // Elt is a special case because it can take variable number of arguments. ve = getEltExpression(childExpr, returnType); + } else if (udf instanceof GenericUDFGrouping) { + ve = getGroupingExpression((GenericUDFGrouping) udf, childExpr, returnType); } else if (udf instanceof GenericUDFBridge) { ve = getGenericUDFBridgeVectorExpression((GenericUDFBridge) udf, childExpr, mode, returnType); @@ -2214,6 +2216,53 @@ private VectorExpression getEltExpression(List childExpr, TypeInfo return vectorElt; } + private VectorExpression getGroupingExpression(GenericUDFGrouping udf, + List childExprs, TypeInfo returnType) + throws HiveException { + + ExprNodeDesc childExpr0 = childExprs.get(0); + if (!(childExpr0 instanceof ExprNodeColumnDesc)) { + return null; + } + ExprNodeColumnDesc groupingIdColDesc = (ExprNodeColumnDesc) childExpr0; + int groupingIdColNum = getInputColumnIndex(groupingIdColDesc.getColumn()); + + final int indexCount = childExprs.size() - 1; + int[] indices = new int[indexCount]; + for (int i = 0; i < indexCount; i++) { + ExprNodeDesc indexChildExpr = childExprs.get(i + 1); + if (!(indexChildExpr instanceof ExprNodeConstantDesc)) { + return null; + } + Object scalarObject = ((ExprNodeConstantDesc) indexChildExpr).getValue(); + final int index; + if (scalarObject instanceof Integer) { + index = (int) scalarObject; + } else if (scalarObject instanceof Long) { + index = (int) ((long) scalarObject); + } else { + return null; + } + indices[i] = index; + } + + final int outputColumnNum = ocm.allocateOutputColumn(returnType); + final VectorExpression ve; + if (indices.length == 1) { + ve = new GroupingColumn(groupingIdColNum, indices[0], outputColumnNum); + } else { + ve = new GroupingColumns(groupingIdColNum, indices, outputColumnNum); + } + + ve.setInputTypeInfos(groupingIdColDesc.getTypeInfo()); + ve.setInputDataTypePhysicalVariations(DataTypePhysicalVariation.NONE); + + ve.setOutputTypeInfo(returnType); + ve.setOutputDataTypePhysicalVariation(DataTypePhysicalVariation.NONE); + + return ve; + } + public enum InConstantType { INT_FAMILY, TIMESTAMP, diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/GroupingColumn.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/GroupingColumn.java new file mode 100644 index 0000000..9bad386 --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/GroupingColumn.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.exec.vector.expressions; + +import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; + +public class GroupingColumn extends MathFuncLongToLong { + private static final long serialVersionUID = 1L; + + private final long mask; + + public GroupingColumn(int inputColumnNum, int index, int outputColumnNum) { + super(inputColumnNum, outputColumnNum); + this.mask = 1L << index; + } + + public GroupingColumn() { + super(); + + // Dummy final assignments. + mask = 0; + } + + @Override + protected long func(long v) { + return (v & mask) == 0 ? 0 : 1; + } + + @Override + public String vectorExpressionParameters() { + return "col " + colNum + ", mask " + mask; + } + + @Override + public VectorExpressionDescriptor.Descriptor getDescriptor() { + return null; // Not applicable. + } +} diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/GroupingColumns.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/GroupingColumns.java new file mode 100644 index 0000000..b59204e --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/GroupingColumns.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.exec.vector.expressions; + +import java.util.Arrays; + +import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; + +public class GroupingColumns extends MathFuncLongToLong { + private static final long serialVersionUID = 1L; + + private final long[] masks; + + public GroupingColumns(int inputColumnNum, int[] indices, int outputColumnNum) { + super(inputColumnNum, outputColumnNum); + final int size = indices.length; + masks = new long[size]; + for (int i = 0; i < size; i++) { + masks[i] = 1L << indices[i]; + } + } + + public GroupingColumns() { + super(); + + // Dummy final assignments. + masks = null; + } + + @Override + protected long func(long v) { + + final int size = masks.length; + final int adjust = size - 1; + long result = 0; + for (int i = 0; i < size; i++) { + if ((v & masks[i]) != 0) { + result += 1L << (adjust - i); + } + } + return result; + } + + @Override + public String vectorExpressionParameters() { + return "col " + colNum + ", masks " + Arrays.toString(masks); + } + + @Override + public VectorExpressionDescriptor.Descriptor getDescriptor() { + return null; // Not applicable. + } +} diff --git ql/src/test/results/clientpositive/llap/vector_groupby_grouping_sets_grouping.q.out ql/src/test/results/clientpositive/llap/vector_groupby_grouping_sets_grouping.q.out index 9501927..d75f2d8 100644 --- ql/src/test/results/clientpositive/llap/vector_groupby_grouping_sets_grouping.q.out +++ ql/src/test/results/clientpositive/llap/vector_groupby_grouping_sets_grouping.q.out @@ -118,7 +118,7 @@ STAGE PLANS: reduceColumnNullOrder: aaa reduceColumnSortOrder: +++ allNative: false - usesVectorUDFAdaptor: true + usesVectorUDFAdaptor: false vectorized: true rowBatchContext: dataColumnCount: 3 @@ -145,7 +145,7 @@ STAGE PLANS: className: VectorSelectOperator native: true projectedOutputColumnNums: [0, 1, 2, 3, 4] - selectExpressions: VectorUDFAdaptor(grouping(_col2, 1)) -> 3:bigint, VectorUDFAdaptor(grouping(_col2, 0)) -> 4:bigint + selectExpressions: GroupingColumn(col 2, mask 2) -> 3:bigint, GroupingColumn(col 2, mask 1) -> 4:bigint Statistics: Num rows: 9 Data size: 72 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -279,7 +279,7 @@ STAGE PLANS: reduceColumnNullOrder: aaa reduceColumnSortOrder: +++ allNative: false - usesVectorUDFAdaptor: true + usesVectorUDFAdaptor: false vectorized: true rowBatchContext: dataColumnCount: 3 @@ -306,7 +306,7 @@ STAGE PLANS: className: VectorSelectOperator native: true projectedOutputColumnNums: [0, 1, 2, 3, 4] - selectExpressions: VectorUDFAdaptor(grouping(_col2, 1)) -> 3:bigint, VectorUDFAdaptor(grouping(_col2, 0)) -> 4:bigint + selectExpressions: GroupingColumn(col 2, mask 2) -> 3:bigint, GroupingColumn(col 2, mask 1) -> 4:bigint Statistics: Num rows: 12 Data size: 96 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -447,7 +447,7 @@ STAGE PLANS: reduceColumnNullOrder: aaa reduceColumnSortOrder: +++ allNative: false - usesVectorUDFAdaptor: true + usesVectorUDFAdaptor: false vectorized: true rowBatchContext: dataColumnCount: 3 @@ -471,7 +471,7 @@ STAGE PLANS: Filter Vectorization: className: VectorFilterOperator native: true - predicateExpression: FilterLongColEqualLongScalar(col 3:bigint, val 1)(children: VectorUDFAdaptor(grouping(_col2, 1)) -> 3:bigint) + predicateExpression: FilterLongColEqualLongScalar(col 3:bigint, val 1)(children: GroupingColumn(col 2, mask 2) -> 3:bigint) predicate: (grouping(_col2, 1) = 1) (type: boolean) Statistics: Num rows: 6 Data size: 48 Basic stats: COMPLETE Column stats: NONE Select Operator @@ -616,7 +616,7 @@ STAGE PLANS: reduceColumnNullOrder: aaa reduceColumnSortOrder: +++ allNative: false - usesVectorUDFAdaptor: true + usesVectorUDFAdaptor: false vectorized: true rowBatchContext: dataColumnCount: 3 @@ -640,7 +640,7 @@ STAGE PLANS: Filter Vectorization: className: VectorFilterOperator native: true - predicateExpression: FilterExprOrExpr(children: FilterLongColEqualLongScalar(col 3:bigint, val 1)(children: VectorUDFAdaptor(grouping(_col2, 1)) -> 3:bigint), FilterLongColEqualLongScalar(col 3:bigint, val 1)(children: VectorUDFAdaptor(grouping(_col2, 0)) -> 3:bigint)) + predicateExpression: FilterExprOrExpr(children: FilterLongColEqualLongScalar(col 3:bigint, val 1)(children: GroupingColumn(col 2, mask 2) -> 3:bigint), FilterLongColEqualLongScalar(col 3:bigint, val 1)(children: GroupingColumn(col 2, mask 1) -> 3:bigint)) predicate: ((grouping(_col2, 0) = 1) or (grouping(_col2, 1) = 1)) (type: boolean) Statistics: Num rows: 12 Data size: 96 Basic stats: COMPLETE Column stats: NONE Select Operator @@ -650,7 +650,7 @@ STAGE PLANS: className: VectorSelectOperator native: true projectedOutputColumnNums: [0, 1, 5, 4] - selectExpressions: LongColAddLongColumn(col 3:bigint, col 4:bigint)(children: VectorUDFAdaptor(grouping(_col2, 1)) -> 3:bigint, VectorUDFAdaptor(grouping(_col2, 0)) -> 4:bigint) -> 5:bigint, IfExprColumnNull(col 3:boolean, col 0:int, null)(children: LongColEqualLongScalar(col 6:bigint, val 1)(children: LongColAddLongColumn(col 3:bigint, col 4:bigint)(children: VectorUDFAdaptor(grouping(_col2, 1)) -> 3:bigint, VectorUDFAdaptor(grouping(_col2, 0)) -> 4:bigint) -> 6:bigint) -> 3:boolean, col 0:int) -> 4:int + selectExpressions: LongColAddLongColumn(col 3:bigint, col 4:bigint)(children: GroupingColumn(col 2, mask 2) -> 3:bigint, GroupingColumn(col 2, mask 1) -> 4:bigint) -> 5:bigint, IfExprColumnNull(col 3:boolean, col 0:int, null)(children: LongColEqualLongScalar(col 6:bigint, val 1)(children: LongColAddLongColumn(col 3:bigint, col 4:bigint)(children: GroupingColumn(col 2, mask 2) -> 3:bigint, GroupingColumn(col 2, mask 1) -> 4:bigint) -> 6:bigint) -> 3:boolean, col 0:int) -> 4:int Statistics: Num rows: 12 Data size: 96 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: bigint), _col3 (type: int) @@ -822,7 +822,7 @@ STAGE PLANS: reduceColumnNullOrder: aaa reduceColumnSortOrder: +++ allNative: false - usesVectorUDFAdaptor: true + usesVectorUDFAdaptor: false vectorized: true rowBatchContext: dataColumnCount: 3 @@ -849,7 +849,7 @@ STAGE PLANS: className: VectorSelectOperator native: true projectedOutputColumnNums: [0, 1, 2, 3, 4] - selectExpressions: VectorUDFAdaptor(grouping(_col2, 1L)) -> 3:bigint, VectorUDFAdaptor(grouping(_col2, 0L)) -> 4:bigint + selectExpressions: GroupingColumn(col 2, mask 2) -> 3:bigint, GroupingColumn(col 2, mask 1) -> 4:bigint Statistics: Num rows: 9 Data size: 72 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -983,7 +983,7 @@ STAGE PLANS: reduceColumnNullOrder: aaa reduceColumnSortOrder: +++ allNative: false - usesVectorUDFAdaptor: true + usesVectorUDFAdaptor: false vectorized: true rowBatchContext: dataColumnCount: 3 @@ -1010,7 +1010,7 @@ STAGE PLANS: className: VectorSelectOperator native: true projectedOutputColumnNums: [0, 1, 2, 3, 4] - selectExpressions: VectorUDFAdaptor(grouping(_col2, 1L)) -> 3:bigint, VectorUDFAdaptor(grouping(_col2, 0L)) -> 4:bigint + selectExpressions: GroupingColumn(col 2, mask 2) -> 3:bigint, GroupingColumn(col 2, mask 1) -> 4:bigint Statistics: Num rows: 12 Data size: 96 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -1119,7 +1119,7 @@ STAGE PLANS: Filter Vectorization: className: VectorFilterOperator native: true - predicateExpression: FilterLongColEqualLongScalar(col 3:bigint, val 1)(children: VectorUDFAdaptor(grouping(_col2, 1L)) -> 3:bigint) + predicateExpression: FilterLongColEqualLongScalar(col 3:bigint, val 1)(children: GroupingColumn(col 2, mask 2) -> 3:bigint) predicate: (grouping(_col2, 1L) = 1) (type: boolean) Statistics: Num rows: 12 Data size: 96 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator @@ -1142,7 +1142,7 @@ STAGE PLANS: featureSupportInUse: [DECIMAL_64] inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat allNative: false - usesVectorUDFAdaptor: true + usesVectorUDFAdaptor: false vectorized: true rowBatchContext: dataColumnCount: 2 @@ -1281,7 +1281,7 @@ STAGE PLANS: Filter Vectorization: className: VectorFilterOperator native: true - predicateExpression: FilterExprOrExpr(children: FilterLongColEqualLongScalar(col 3:bigint, val 1)(children: VectorUDFAdaptor(grouping(_col2, 1L)) -> 3:bigint), FilterLongColEqualLongScalar(col 3:bigint, val 1)(children: VectorUDFAdaptor(grouping(_col2, 0L)) -> 3:bigint)) + predicateExpression: FilterExprOrExpr(children: FilterLongColEqualLongScalar(col 3:bigint, val 1)(children: GroupingColumn(col 2, mask 2) -> 3:bigint), FilterLongColEqualLongScalar(col 3:bigint, val 1)(children: GroupingColumn(col 2, mask 1) -> 3:bigint)) predicate: ((grouping(_col2, 0L) = 1) or (grouping(_col2, 1L) = 1)) (type: boolean) Statistics: Num rows: 24 Data size: 192 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator @@ -1304,7 +1304,7 @@ STAGE PLANS: featureSupportInUse: [DECIMAL_64] inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat allNative: false - usesVectorUDFAdaptor: true + usesVectorUDFAdaptor: false vectorized: true rowBatchContext: dataColumnCount: 2 @@ -1320,7 +1320,7 @@ STAGE PLANS: reduceColumnNullOrder: aaa reduceColumnSortOrder: +++ allNative: false - usesVectorUDFAdaptor: true + usesVectorUDFAdaptor: false vectorized: true rowBatchContext: dataColumnCount: 3 @@ -1347,7 +1347,7 @@ STAGE PLANS: className: VectorSelectOperator native: true projectedOutputColumnNums: [0, 1, 5] - selectExpressions: LongColAddLongColumn(col 3:bigint, col 4:bigint)(children: VectorUDFAdaptor(grouping(_col2, 1L)) -> 3:bigint, VectorUDFAdaptor(grouping(_col2, 0L)) -> 4:bigint) -> 5:bigint + selectExpressions: LongColAddLongColumn(col 3:bigint, col 4:bigint)(children: GroupingColumn(col 2, mask 2) -> 3:bigint, GroupingColumn(col 2, mask 1) -> 4:bigint) -> 5:bigint Statistics: Num rows: 12 Data size: 96 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: bigint), CASE WHEN ((_col2 = 1L)) THEN (_col0) END (type: int) @@ -1983,7 +1983,7 @@ STAGE PLANS: reduceColumnNullOrder: aaa reduceColumnSortOrder: +++ allNative: false - usesVectorUDFAdaptor: true + usesVectorUDFAdaptor: false vectorized: true rowBatchContext: dataColumnCount: 3 @@ -2010,7 +2010,7 @@ STAGE PLANS: className: VectorSelectOperator native: true projectedOutputColumnNums: [0, 1, 2, 3] - selectExpressions: VectorUDFAdaptor(grouping(_col2, 1L, 0L)) -> 3:bigint + selectExpressions: GroupingColumns(col 2, masks [2, 1]) -> 3:bigint Statistics: Num rows: 12 Data size: 96 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -2149,7 +2149,7 @@ STAGE PLANS: reduceColumnNullOrder: aaa reduceColumnSortOrder: +++ allNative: false - usesVectorUDFAdaptor: true + usesVectorUDFAdaptor: false vectorized: true rowBatchContext: dataColumnCount: 3 @@ -2176,7 +2176,7 @@ STAGE PLANS: className: VectorSelectOperator native: true projectedOutputColumnNums: [0, 1, 2, 3] - selectExpressions: VectorUDFAdaptor(grouping(_col2, 0L, 1L)) -> 3:bigint + selectExpressions: GroupingColumns(col 2, masks [1, 2]) -> 3:bigint Statistics: Num rows: 12 Data size: 96 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -2315,7 +2315,7 @@ STAGE PLANS: reduceColumnNullOrder: aaa reduceColumnSortOrder: +++ allNative: false - usesVectorUDFAdaptor: true + usesVectorUDFAdaptor: false vectorized: true rowBatchContext: dataColumnCount: 3 @@ -2342,7 +2342,7 @@ STAGE PLANS: className: VectorSelectOperator native: true projectedOutputColumnNums: [0, 1, 2, 3] - selectExpressions: VectorUDFAdaptor(grouping(_col2, 1L, 0L)) -> 3:bigint + selectExpressions: GroupingColumns(col 2, masks [2, 1]) -> 3:bigint Statistics: Num rows: 9 Data size: 72 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -2476,7 +2476,7 @@ STAGE PLANS: reduceColumnNullOrder: aaa reduceColumnSortOrder: +++ allNative: false - usesVectorUDFAdaptor: true + usesVectorUDFAdaptor: false vectorized: true rowBatchContext: dataColumnCount: 3 @@ -2503,7 +2503,7 @@ STAGE PLANS: className: VectorSelectOperator native: true projectedOutputColumnNums: [0, 1, 2, 3] - selectExpressions: VectorUDFAdaptor(grouping(_col2, 0L, 1L)) -> 3:bigint + selectExpressions: GroupingColumns(col 2, masks [1, 2]) -> 3:bigint Statistics: Num rows: 9 Data size: 72 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false diff --git ql/src/test/results/clientpositive/perf/spark/query27.q.out ql/src/test/results/clientpositive/perf/spark/query27.q.out index 82b4dbf..bad5207 100644 --- ql/src/test/results/clientpositive/perf/spark/query27.q.out +++ ql/src/test/results/clientpositive/perf/spark/query27.q.out @@ -371,7 +371,7 @@ STAGE PLANS: enabled: true enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine spark IN [tez, spark] IS true allNative: false - usesVectorUDFAdaptor: true + usesVectorUDFAdaptor: false vectorized: true Reduce Operator Tree: Group By Operator @@ -395,7 +395,7 @@ STAGE PLANS: className: VectorSelectOperator native: true projectedOutputColumnNums: [0, 1, 11, 12, 14, 15, 16] - selectExpressions: VectorUDFAdaptor(grouping(_col2, 0)) -> 11:bigint, LongColDivideLongColumn(col 3:bigint, col 4:bigint) -> 12:double, DecimalColDivideDecimalColumn(col 5:decimal(17,2), col 13:decimal(19,0))(children: CastLongToDecimal(col 6:bigint) -> 13:decimal(19,0)) -> 14:decimal(37,22), DecimalColDivideDecimalColumn(col 7:decimal(17,2), col 13:decimal(19,0))(children: CastLongToDecimal(col 8:bigint) -> 13:decimal(19,0)) -> 15:decimal(37,22), DecimalColDivideDecimalColumn(col 9:decimal(17,2), col 13:decimal(19,0))(children: CastLongToDecimal(col 10:bigint) -> 13:decimal(19,0)) -> 16:decimal(37,22) + selectExpressions: GroupingColumn(col 2, mask 1) -> 11:bigint, LongColDivideLongColumn(col 3:bigint, col 4:bigint) -> 12:double, DecimalColDivideDecimalColumn(col 5:decimal(17,2), col 13:decimal(19,0))(children: CastLongToDecimal(col 6:bigint) -> 13:decimal(19,0)) -> 14:decimal(37,22), DecimalColDivideDecimalColumn(col 7:decimal(17,2), col 13:decimal(19,0))(children: CastLongToDecimal(col 8:bigint) -> 13:decimal(19,0)) -> 15:decimal(37,22), DecimalColDivideDecimalColumn(col 9:decimal(17,2), col 13:decimal(19,0))(children: CastLongToDecimal(col 10:bigint) -> 13:decimal(19,0)) -> 16:decimal(37,22) Statistics: Num rows: 1264972921 Data size: 111596278389 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) diff --git ql/src/test/results/clientpositive/perf/spark/query36.q.out ql/src/test/results/clientpositive/perf/spark/query36.q.out index 5ff3d9d..ea7639d 100644 --- ql/src/test/results/clientpositive/perf/spark/query36.q.out +++ ql/src/test/results/clientpositive/perf/spark/query36.q.out @@ -321,7 +321,7 @@ STAGE PLANS: enabled: true enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine spark IN [tez, spark] IS true allNative: false - usesVectorUDFAdaptor: true + usesVectorUDFAdaptor: false vectorized: true Reduce Operator Tree: Group By Operator @@ -352,7 +352,7 @@ STAGE PLANS: Map-reduce partition columns: (grouping(_col4, 1) + grouping(_col4, 0)) (type: bigint), CASE WHEN ((grouping(_col4, 0) = 0)) THEN (_col0) ELSE (CAST( null AS STRING)) END (type: string) Reduce Sink Vectorization: className: VectorReduceSinkObjectHashOperator - keyExpressions: LongColAddLongColumn(col 5:bigint, col 6:bigint)(children: VectorUDFAdaptor(grouping(_col4, 1)) -> 5:bigint, VectorUDFAdaptor(grouping(_col4, 0)) -> 6:bigint) -> 7:bigint, IfExprColumnNull(col 6:boolean, col 0:string, null)(children: LongColEqualLongScalar(col 5:bigint, val 0)(children: VectorUDFAdaptor(grouping(_col4, 0)) -> 5:bigint) -> 6:boolean, col 0:string) -> 8:string, DecimalColDivideDecimalColumn(col 3:decimal(17,2), col 4:decimal(17,2)) -> 9:decimal(37,20) + keyExpressions: LongColAddLongColumn(col 5:bigint, col 6:bigint)(children: GroupingColumn(col 2, mask 2) -> 5:bigint, GroupingColumn(col 2, mask 1) -> 6:bigint) -> 7:bigint, IfExprColumnNull(col 6:boolean, col 0:string, null)(children: LongColEqualLongScalar(col 5:bigint, val 0)(children: GroupingColumn(col 2, mask 1) -> 5:bigint) -> 6:boolean, col 0:string) -> 8:string, DecimalColDivideDecimalColumn(col 3:decimal(17,2), col 4:decimal(17,2)) -> 9:decimal(37,20) native: true nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine spark IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true Statistics: Num rows: 1149975358 Data size: 101451159969 Basic stats: COMPLETE Column stats: NONE @@ -363,7 +363,7 @@ STAGE PLANS: enabled: true enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine spark IN [tez, spark] IS true allNative: true - usesVectorUDFAdaptor: true + usesVectorUDFAdaptor: false vectorized: true Reduce Operator Tree: Select Operator @@ -401,7 +401,7 @@ STAGE PLANS: functionNames: [rank] native: true orderExpressions: [DecimalColDivideDecimalColumn(col 5:decimal(17,2), col 6:decimal(17,2)) -> 13:decimal(37,20)] - partitionExpressions: [LongColAddLongColumn(col 9:bigint, col 10:bigint)(children: VectorUDFAdaptor(grouping(_col4, 1)) -> 9:bigint, VectorUDFAdaptor(grouping(_col4, 0)) -> 10:bigint) -> 11:bigint, IfExprColumnNull(col 10:boolean, col 3:string, null)(children: LongColEqualLongScalar(col 9:bigint, val 0)(children: VectorUDFAdaptor(grouping(_col4, 0)) -> 9:bigint) -> 10:boolean, col 3:string) -> 12:string] + partitionExpressions: [LongColAddLongColumn(col 9:bigint, col 10:bigint)(children: GroupingColumn(col 7, mask 2) -> 9:bigint, GroupingColumn(col 7, mask 1) -> 10:bigint) -> 11:bigint, IfExprColumnNull(col 10:boolean, col 3:string, null)(children: LongColEqualLongScalar(col 9:bigint, val 0)(children: GroupingColumn(col 7, mask 1) -> 9:bigint) -> 10:boolean, col 3:string) -> 12:string] Statistics: Num rows: 1149975358 Data size: 101451159969 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: (_col2 / _col3) (type: decimal(37,20)), _col0 (type: string), _col1 (type: string), (grouping(_col4, 1) + grouping(_col4, 0)) (type: bigint), rank_window_0 (type: int), CASE WHEN (((grouping(_col4, 1) + grouping(_col4, 0)) = 0)) THEN (_col0) ELSE (null) END (type: string) @@ -410,7 +410,7 @@ STAGE PLANS: className: VectorSelectOperator native: true projectedOutputColumnNums: [15, 3, 4, 17, 8, 19] - selectExpressions: DecimalColDivideDecimalColumn(col 5:decimal(17,2), col 6:decimal(17,2)) -> 15:decimal(37,20), LongColAddLongColumn(col 9:bigint, col 16:bigint)(children: VectorUDFAdaptor(grouping(_col4, 1)) -> 9:bigint, VectorUDFAdaptor(grouping(_col4, 0)) -> 16:bigint) -> 17:bigint, IfExprColumnNull(col 9:boolean, col 3:string, null)(children: LongColEqualLongScalar(col 18:bigint, val 0)(children: LongColAddLongColumn(col 9:bigint, col 16:bigint)(children: VectorUDFAdaptor(grouping(_col4, 1)) -> 9:bigint, VectorUDFAdaptor(grouping(_col4, 0)) -> 16:bigint) -> 18:bigint) -> 9:boolean, col 3:string) -> 19:string + selectExpressions: DecimalColDivideDecimalColumn(col 5:decimal(17,2), col 6:decimal(17,2)) -> 15:decimal(37,20), LongColAddLongColumn(col 9:bigint, col 16:bigint)(children: GroupingColumn(col 7, mask 2) -> 9:bigint, GroupingColumn(col 7, mask 1) -> 16:bigint) -> 17:bigint, IfExprColumnNull(col 9:boolean, col 3:string, null)(children: LongColEqualLongScalar(col 18:bigint, val 0)(children: LongColAddLongColumn(col 9:bigint, col 16:bigint)(children: GroupingColumn(col 7, mask 2) -> 9:bigint, GroupingColumn(col 7, mask 1) -> 16:bigint) -> 18:bigint) -> 9:boolean, col 3:string) -> 19:string Statistics: Num rows: 1149975358 Data size: 101451159969 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col3 (type: bigint), _col5 (type: string), _col4 (type: int) diff --git ql/src/test/results/clientpositive/perf/spark/query70.q.out ql/src/test/results/clientpositive/perf/spark/query70.q.out index b719cdb..ee0e165 100644 --- ql/src/test/results/clientpositive/perf/spark/query70.q.out +++ ql/src/test/results/clientpositive/perf/spark/query70.q.out @@ -573,7 +573,7 @@ STAGE PLANS: enabled: true enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine spark IN [tez, spark] IS true allNative: false - usesVectorUDFAdaptor: true + usesVectorUDFAdaptor: false vectorized: true Reduce Operator Tree: Group By Operator @@ -604,7 +604,7 @@ STAGE PLANS: Map-reduce partition columns: (grouping(_col3, 1) + grouping(_col3, 0)) (type: bigint), CASE WHEN ((grouping(_col3, 0) = 0)) THEN (_col0) ELSE (CAST( null AS STRING)) END (type: string) Reduce Sink Vectorization: className: VectorReduceSinkObjectHashOperator - keyExpressions: LongColAddLongColumn(col 4:bigint, col 5:bigint)(children: VectorUDFAdaptor(grouping(_col3, 1)) -> 4:bigint, VectorUDFAdaptor(grouping(_col3, 0)) -> 5:bigint) -> 6:bigint, IfExprColumnNull(col 5:boolean, col 0:string, null)(children: LongColEqualLongScalar(col 4:bigint, val 0)(children: VectorUDFAdaptor(grouping(_col3, 0)) -> 4:bigint) -> 5:boolean, col 0:string) -> 7:string + keyExpressions: LongColAddLongColumn(col 4:bigint, col 5:bigint)(children: GroupingColumn(col 2, mask 2) -> 4:bigint, GroupingColumn(col 2, mask 1) -> 5:bigint) -> 6:bigint, IfExprColumnNull(col 5:boolean, col 0:string, null)(children: LongColEqualLongScalar(col 4:bigint, val 0)(children: GroupingColumn(col 2, mask 1) -> 4:bigint) -> 5:boolean, col 0:string) -> 7:string native: true nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine spark IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true Statistics: Num rows: 1149975358 Data size: 101451159969 Basic stats: COMPLETE Column stats: NONE @@ -615,7 +615,7 @@ STAGE PLANS: enabled: true enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine spark IN [tez, spark] IS true allNative: true - usesVectorUDFAdaptor: true + usesVectorUDFAdaptor: false vectorized: true Reduce Operator Tree: Select Operator @@ -653,7 +653,7 @@ STAGE PLANS: functionNames: [rank] native: true orderExpressions: [col 2:decimal(17,2)] - partitionExpressions: [LongColAddLongColumn(col 7:bigint, col 8:bigint)(children: VectorUDFAdaptor(grouping(_col3, 1)) -> 7:bigint, VectorUDFAdaptor(grouping(_col3, 0)) -> 8:bigint) -> 9:bigint, IfExprColumnNull(col 8:boolean, col 3:string, null)(children: LongColEqualLongScalar(col 7:bigint, val 0)(children: VectorUDFAdaptor(grouping(_col3, 0)) -> 7:bigint) -> 8:boolean, col 3:string) -> 10:string] + partitionExpressions: [LongColAddLongColumn(col 7:bigint, col 8:bigint)(children: GroupingColumn(col 5, mask 2) -> 7:bigint, GroupingColumn(col 5, mask 1) -> 8:bigint) -> 9:bigint, IfExprColumnNull(col 8:boolean, col 3:string, null)(children: LongColEqualLongScalar(col 7:bigint, val 0)(children: GroupingColumn(col 5, mask 1) -> 7:bigint) -> 8:boolean, col 3:string) -> 10:string] Statistics: Num rows: 1149975358 Data size: 101451159969 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: decimal(17,2)), _col0 (type: string), _col1 (type: string), (grouping(_col3, 1) + grouping(_col3, 0)) (type: bigint), rank_window_0 (type: int), CASE WHEN (((grouping(_col3, 1) + grouping(_col3, 0)) = 0)) THEN (_col0) ELSE (null) END (type: string) @@ -662,7 +662,7 @@ STAGE PLANS: className: VectorSelectOperator native: true projectedOutputColumnNums: [2, 3, 4, 12, 6, 14] - selectExpressions: LongColAddLongColumn(col 7:bigint, col 11:bigint)(children: VectorUDFAdaptor(grouping(_col3, 1)) -> 7:bigint, VectorUDFAdaptor(grouping(_col3, 0)) -> 11:bigint) -> 12:bigint, IfExprColumnNull(col 7:boolean, col 3:string, null)(children: LongColEqualLongScalar(col 13:bigint, val 0)(children: LongColAddLongColumn(col 7:bigint, col 11:bigint)(children: VectorUDFAdaptor(grouping(_col3, 1)) -> 7:bigint, VectorUDFAdaptor(grouping(_col3, 0)) -> 11:bigint) -> 13:bigint) -> 7:boolean, col 3:string) -> 14:string + selectExpressions: LongColAddLongColumn(col 7:bigint, col 11:bigint)(children: GroupingColumn(col 5, mask 2) -> 7:bigint, GroupingColumn(col 5, mask 1) -> 11:bigint) -> 12:bigint, IfExprColumnNull(col 7:boolean, col 3:string, null)(children: LongColEqualLongScalar(col 13:bigint, val 0)(children: LongColAddLongColumn(col 7:bigint, col 11:bigint)(children: GroupingColumn(col 5, mask 2) -> 7:bigint, GroupingColumn(col 5, mask 1) -> 11:bigint) -> 13:bigint) -> 7:boolean, col 3:string) -> 14:string Statistics: Num rows: 1149975358 Data size: 101451159969 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col3 (type: bigint), _col5 (type: string), _col4 (type: int) diff --git ql/src/test/results/clientpositive/perf/spark/query86.q.out ql/src/test/results/clientpositive/perf/spark/query86.q.out index 231d57d..c2d34e0 100644 --- ql/src/test/results/clientpositive/perf/spark/query86.q.out +++ ql/src/test/results/clientpositive/perf/spark/query86.q.out @@ -253,7 +253,7 @@ STAGE PLANS: enabled: true enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine spark IN [tez, spark] IS true allNative: false - usesVectorUDFAdaptor: true + usesVectorUDFAdaptor: false vectorized: true Reduce Operator Tree: Group By Operator @@ -284,7 +284,7 @@ STAGE PLANS: Map-reduce partition columns: (grouping(_col3, 1) + grouping(_col3, 0)) (type: bigint), CASE WHEN ((grouping(_col3, 0) = 0)) THEN (_col0) ELSE (CAST( null AS STRING)) END (type: string) Reduce Sink Vectorization: className: VectorReduceSinkObjectHashOperator - keyExpressions: LongColAddLongColumn(col 4:bigint, col 5:bigint)(children: VectorUDFAdaptor(grouping(_col3, 1)) -> 4:bigint, VectorUDFAdaptor(grouping(_col3, 0)) -> 5:bigint) -> 6:bigint, IfExprColumnNull(col 5:boolean, col 0:string, null)(children: LongColEqualLongScalar(col 4:bigint, val 0)(children: VectorUDFAdaptor(grouping(_col3, 0)) -> 4:bigint) -> 5:boolean, col 0:string) -> 7:string + keyExpressions: LongColAddLongColumn(col 4:bigint, col 5:bigint)(children: GroupingColumn(col 2, mask 2) -> 4:bigint, GroupingColumn(col 2, mask 1) -> 5:bigint) -> 6:bigint, IfExprColumnNull(col 5:boolean, col 0:string, null)(children: LongColEqualLongScalar(col 4:bigint, val 0)(children: GroupingColumn(col 2, mask 1) -> 4:bigint) -> 5:boolean, col 0:string) -> 7:string native: true nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine spark IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true Statistics: Num rows: 261364852 Data size: 35538061226 Basic stats: COMPLETE Column stats: NONE @@ -295,7 +295,7 @@ STAGE PLANS: enabled: true enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine spark IN [tez, spark] IS true allNative: true - usesVectorUDFAdaptor: true + usesVectorUDFAdaptor: false vectorized: true Reduce Operator Tree: Select Operator @@ -333,7 +333,7 @@ STAGE PLANS: functionNames: [rank] native: true orderExpressions: [col 2:decimal(17,2)] - partitionExpressions: [LongColAddLongColumn(col 7:bigint, col 8:bigint)(children: VectorUDFAdaptor(grouping(_col3, 1)) -> 7:bigint, VectorUDFAdaptor(grouping(_col3, 0)) -> 8:bigint) -> 9:bigint, IfExprColumnNull(col 8:boolean, col 3:string, null)(children: LongColEqualLongScalar(col 7:bigint, val 0)(children: VectorUDFAdaptor(grouping(_col3, 0)) -> 7:bigint) -> 8:boolean, col 3:string) -> 10:string] + partitionExpressions: [LongColAddLongColumn(col 7:bigint, col 8:bigint)(children: GroupingColumn(col 5, mask 2) -> 7:bigint, GroupingColumn(col 5, mask 1) -> 8:bigint) -> 9:bigint, IfExprColumnNull(col 8:boolean, col 3:string, null)(children: LongColEqualLongScalar(col 7:bigint, val 0)(children: GroupingColumn(col 5, mask 1) -> 7:bigint) -> 8:boolean, col 3:string) -> 10:string] Statistics: Num rows: 261364852 Data size: 35538061226 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: decimal(17,2)), _col0 (type: string), _col1 (type: string), (grouping(_col3, 1) + grouping(_col3, 0)) (type: bigint), rank_window_0 (type: int), CASE WHEN (((grouping(_col3, 1) + grouping(_col3, 0)) = 0)) THEN (_col0) ELSE (null) END (type: string) @@ -342,7 +342,7 @@ STAGE PLANS: className: VectorSelectOperator native: true projectedOutputColumnNums: [2, 3, 4, 12, 6, 14] - selectExpressions: LongColAddLongColumn(col 7:bigint, col 11:bigint)(children: VectorUDFAdaptor(grouping(_col3, 1)) -> 7:bigint, VectorUDFAdaptor(grouping(_col3, 0)) -> 11:bigint) -> 12:bigint, IfExprColumnNull(col 7:boolean, col 3:string, null)(children: LongColEqualLongScalar(col 13:bigint, val 0)(children: LongColAddLongColumn(col 7:bigint, col 11:bigint)(children: VectorUDFAdaptor(grouping(_col3, 1)) -> 7:bigint, VectorUDFAdaptor(grouping(_col3, 0)) -> 11:bigint) -> 13:bigint) -> 7:boolean, col 3:string) -> 14:string + selectExpressions: LongColAddLongColumn(col 7:bigint, col 11:bigint)(children: GroupingColumn(col 5, mask 2) -> 7:bigint, GroupingColumn(col 5, mask 1) -> 11:bigint) -> 12:bigint, IfExprColumnNull(col 7:boolean, col 3:string, null)(children: LongColEqualLongScalar(col 13:bigint, val 0)(children: LongColAddLongColumn(col 7:bigint, col 11:bigint)(children: GroupingColumn(col 5, mask 2) -> 7:bigint, GroupingColumn(col 5, mask 1) -> 11:bigint) -> 13:bigint) -> 7:boolean, col 3:string) -> 14:string Statistics: Num rows: 261364852 Data size: 35538061226 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col3 (type: bigint), _col5 (type: string), _col4 (type: int) diff --git ql/src/test/results/clientpositive/perf/tez/query27.q.out ql/src/test/results/clientpositive/perf/tez/query27.q.out index 5bb57cb..b39a4ba 100644 --- ql/src/test/results/clientpositive/perf/tez/query27.q.out +++ ql/src/test/results/clientpositive/perf/tez/query27.q.out @@ -560,7 +560,7 @@ STAGE PLANS: enabled: true enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true allNative: false - usesVectorUDFAdaptor: true + usesVectorUDFAdaptor: false vectorized: true Reduce Operator Tree: Group By Operator @@ -584,7 +584,7 @@ STAGE PLANS: className: VectorSelectOperator native: true projectedOutputColumnNums: [0, 1, 11, 12, 14, 15, 16] - selectExpressions: VectorUDFAdaptor(grouping(_col2, 0)) -> 11:bigint, LongColDivideLongColumn(col 3:bigint, col 4:bigint) -> 12:double, DecimalColDivideDecimalColumn(col 5:decimal(17,2), col 13:decimal(19,0))(children: CastLongToDecimal(col 6:bigint) -> 13:decimal(19,0)) -> 14:decimal(37,22), DecimalColDivideDecimalColumn(col 7:decimal(17,2), col 13:decimal(19,0))(children: CastLongToDecimal(col 8:bigint) -> 13:decimal(19,0)) -> 15:decimal(37,22), DecimalColDivideDecimalColumn(col 9:decimal(17,2), col 13:decimal(19,0))(children: CastLongToDecimal(col 10:bigint) -> 13:decimal(19,0)) -> 16:decimal(37,22) + selectExpressions: GroupingColumn(col 2, mask 1) -> 11:bigint, LongColDivideLongColumn(col 3:bigint, col 4:bigint) -> 12:double, DecimalColDivideDecimalColumn(col 5:decimal(17,2), col 13:decimal(19,0))(children: CastLongToDecimal(col 6:bigint) -> 13:decimal(19,0)) -> 14:decimal(37,22), DecimalColDivideDecimalColumn(col 7:decimal(17,2), col 13:decimal(19,0))(children: CastLongToDecimal(col 8:bigint) -> 13:decimal(19,0)) -> 15:decimal(37,22), DecimalColDivideDecimalColumn(col 9:decimal(17,2), col 13:decimal(19,0))(children: CastLongToDecimal(col 10:bigint) -> 13:decimal(19,0)) -> 16:decimal(37,22) Statistics: Num rows: 1264972921 Data size: 111596278389 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) diff --git ql/src/test/results/clientpositive/perf/tez/query36.q.out ql/src/test/results/clientpositive/perf/tez/query36.q.out index 92fd3ce..1be2a4d 100644 --- ql/src/test/results/clientpositive/perf/tez/query36.q.out +++ ql/src/test/results/clientpositive/perf/tez/query36.q.out @@ -451,7 +451,7 @@ STAGE PLANS: enabled: true enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true allNative: false - usesVectorUDFAdaptor: true + usesVectorUDFAdaptor: false vectorized: true Reduce Operator Tree: Group By Operator @@ -482,7 +482,7 @@ STAGE PLANS: Map-reduce partition columns: (grouping(_col4, 1) + grouping(_col4, 0)) (type: bigint), CASE WHEN ((grouping(_col4, 0) = 0)) THEN (_col0) ELSE (CAST( null AS STRING)) END (type: string) Reduce Sink Vectorization: className: VectorReduceSinkObjectHashOperator - keyExpressions: LongColAddLongColumn(col 5:bigint, col 6:bigint)(children: VectorUDFAdaptor(grouping(_col4, 1)) -> 5:bigint, VectorUDFAdaptor(grouping(_col4, 0)) -> 6:bigint) -> 7:bigint, IfExprColumnNull(col 6:boolean, col 0:string, null)(children: LongColEqualLongScalar(col 5:bigint, val 0)(children: VectorUDFAdaptor(grouping(_col4, 0)) -> 5:bigint) -> 6:boolean, col 0:string) -> 8:string, DecimalColDivideDecimalColumn(col 3:decimal(17,2), col 4:decimal(17,2)) -> 9:decimal(37,20) + keyExpressions: LongColAddLongColumn(col 5:bigint, col 6:bigint)(children: GroupingColumn(col 2, mask 2) -> 5:bigint, GroupingColumn(col 2, mask 1) -> 6:bigint) -> 7:bigint, IfExprColumnNull(col 6:boolean, col 0:string, null)(children: LongColEqualLongScalar(col 5:bigint, val 0)(children: GroupingColumn(col 2, mask 1) -> 5:bigint) -> 6:boolean, col 0:string) -> 8:string, DecimalColDivideDecimalColumn(col 3:decimal(17,2), col 4:decimal(17,2)) -> 9:decimal(37,20) native: true nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true Statistics: Num rows: 1149975358 Data size: 101451159969 Basic stats: COMPLETE Column stats: NONE @@ -493,7 +493,7 @@ STAGE PLANS: enabled: true enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true allNative: true - usesVectorUDFAdaptor: true + usesVectorUDFAdaptor: false vectorized: true Reduce Operator Tree: Select Operator @@ -531,7 +531,7 @@ STAGE PLANS: functionNames: [rank] native: true orderExpressions: [DecimalColDivideDecimalColumn(col 5:decimal(17,2), col 6:decimal(17,2)) -> 13:decimal(37,20)] - partitionExpressions: [LongColAddLongColumn(col 9:bigint, col 10:bigint)(children: VectorUDFAdaptor(grouping(_col4, 1)) -> 9:bigint, VectorUDFAdaptor(grouping(_col4, 0)) -> 10:bigint) -> 11:bigint, IfExprColumnNull(col 10:boolean, col 3:string, null)(children: LongColEqualLongScalar(col 9:bigint, val 0)(children: VectorUDFAdaptor(grouping(_col4, 0)) -> 9:bigint) -> 10:boolean, col 3:string) -> 12:string] + partitionExpressions: [LongColAddLongColumn(col 9:bigint, col 10:bigint)(children: GroupingColumn(col 7, mask 2) -> 9:bigint, GroupingColumn(col 7, mask 1) -> 10:bigint) -> 11:bigint, IfExprColumnNull(col 10:boolean, col 3:string, null)(children: LongColEqualLongScalar(col 9:bigint, val 0)(children: GroupingColumn(col 7, mask 1) -> 9:bigint) -> 10:boolean, col 3:string) -> 12:string] Statistics: Num rows: 1149975358 Data size: 101451159969 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: (_col2 / _col3) (type: decimal(37,20)), _col0 (type: string), _col1 (type: string), (grouping(_col4, 1) + grouping(_col4, 0)) (type: bigint), rank_window_0 (type: int), CASE WHEN (((grouping(_col4, 1) + grouping(_col4, 0)) = 0)) THEN (_col0) ELSE (null) END (type: string) @@ -540,7 +540,7 @@ STAGE PLANS: className: VectorSelectOperator native: true projectedOutputColumnNums: [15, 3, 4, 17, 8, 19] - selectExpressions: DecimalColDivideDecimalColumn(col 5:decimal(17,2), col 6:decimal(17,2)) -> 15:decimal(37,20), LongColAddLongColumn(col 9:bigint, col 16:bigint)(children: VectorUDFAdaptor(grouping(_col4, 1)) -> 9:bigint, VectorUDFAdaptor(grouping(_col4, 0)) -> 16:bigint) -> 17:bigint, IfExprColumnNull(col 9:boolean, col 3:string, null)(children: LongColEqualLongScalar(col 18:bigint, val 0)(children: LongColAddLongColumn(col 9:bigint, col 16:bigint)(children: VectorUDFAdaptor(grouping(_col4, 1)) -> 9:bigint, VectorUDFAdaptor(grouping(_col4, 0)) -> 16:bigint) -> 18:bigint) -> 9:boolean, col 3:string) -> 19:string + selectExpressions: DecimalColDivideDecimalColumn(col 5:decimal(17,2), col 6:decimal(17,2)) -> 15:decimal(37,20), LongColAddLongColumn(col 9:bigint, col 16:bigint)(children: GroupingColumn(col 7, mask 2) -> 9:bigint, GroupingColumn(col 7, mask 1) -> 16:bigint) -> 17:bigint, IfExprColumnNull(col 9:boolean, col 3:string, null)(children: LongColEqualLongScalar(col 18:bigint, val 0)(children: LongColAddLongColumn(col 9:bigint, col 16:bigint)(children: GroupingColumn(col 7, mask 2) -> 9:bigint, GroupingColumn(col 7, mask 1) -> 16:bigint) -> 18:bigint) -> 9:boolean, col 3:string) -> 19:string Statistics: Num rows: 1149975358 Data size: 101451159969 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col3 (type: bigint), _col5 (type: string), _col4 (type: int) diff --git ql/src/test/results/clientpositive/perf/tez/query70.q.out ql/src/test/results/clientpositive/perf/tez/query70.q.out index 20a6d9a..97d21e4 100644 --- ql/src/test/results/clientpositive/perf/tez/query70.q.out +++ ql/src/test/results/clientpositive/perf/tez/query70.q.out @@ -494,7 +494,7 @@ STAGE PLANS: enabled: true enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true allNative: false - usesVectorUDFAdaptor: true + usesVectorUDFAdaptor: false vectorized: true Reduce Operator Tree: Group By Operator @@ -525,7 +525,7 @@ STAGE PLANS: Map-reduce partition columns: (grouping(_col3, 1) + grouping(_col3, 0)) (type: bigint), CASE WHEN ((grouping(_col3, 0) = 0)) THEN (_col0) ELSE (CAST( null AS STRING)) END (type: string) Reduce Sink Vectorization: className: VectorReduceSinkObjectHashOperator - keyExpressions: LongColAddLongColumn(col 4:bigint, col 5:bigint)(children: VectorUDFAdaptor(grouping(_col3, 1)) -> 4:bigint, VectorUDFAdaptor(grouping(_col3, 0)) -> 5:bigint) -> 6:bigint, IfExprColumnNull(col 5:boolean, col 0:string, null)(children: LongColEqualLongScalar(col 4:bigint, val 0)(children: VectorUDFAdaptor(grouping(_col3, 0)) -> 4:bigint) -> 5:boolean, col 0:string) -> 7:string + keyExpressions: LongColAddLongColumn(col 4:bigint, col 5:bigint)(children: GroupingColumn(col 2, mask 2) -> 4:bigint, GroupingColumn(col 2, mask 1) -> 5:bigint) -> 6:bigint, IfExprColumnNull(col 5:boolean, col 0:string, null)(children: LongColEqualLongScalar(col 4:bigint, val 0)(children: GroupingColumn(col 2, mask 1) -> 4:bigint) -> 5:boolean, col 0:string) -> 7:string native: true nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true Statistics: Num rows: 1149975358 Data size: 101451159969 Basic stats: COMPLETE Column stats: NONE @@ -536,7 +536,7 @@ STAGE PLANS: enabled: true enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true allNative: true - usesVectorUDFAdaptor: true + usesVectorUDFAdaptor: false vectorized: true Reduce Operator Tree: Select Operator @@ -574,7 +574,7 @@ STAGE PLANS: functionNames: [rank] native: true orderExpressions: [col 2:decimal(17,2)] - partitionExpressions: [LongColAddLongColumn(col 7:bigint, col 8:bigint)(children: VectorUDFAdaptor(grouping(_col3, 1)) -> 7:bigint, VectorUDFAdaptor(grouping(_col3, 0)) -> 8:bigint) -> 9:bigint, IfExprColumnNull(col 8:boolean, col 3:string, null)(children: LongColEqualLongScalar(col 7:bigint, val 0)(children: VectorUDFAdaptor(grouping(_col3, 0)) -> 7:bigint) -> 8:boolean, col 3:string) -> 10:string] + partitionExpressions: [LongColAddLongColumn(col 7:bigint, col 8:bigint)(children: GroupingColumn(col 5, mask 2) -> 7:bigint, GroupingColumn(col 5, mask 1) -> 8:bigint) -> 9:bigint, IfExprColumnNull(col 8:boolean, col 3:string, null)(children: LongColEqualLongScalar(col 7:bigint, val 0)(children: GroupingColumn(col 5, mask 1) -> 7:bigint) -> 8:boolean, col 3:string) -> 10:string] Statistics: Num rows: 1149975358 Data size: 101451159969 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: decimal(17,2)), _col0 (type: string), _col1 (type: string), (grouping(_col3, 1) + grouping(_col3, 0)) (type: bigint), rank_window_0 (type: int), CASE WHEN (((grouping(_col3, 1) + grouping(_col3, 0)) = 0)) THEN (_col0) ELSE (null) END (type: string) @@ -583,7 +583,7 @@ STAGE PLANS: className: VectorSelectOperator native: true projectedOutputColumnNums: [2, 3, 4, 12, 6, 14] - selectExpressions: LongColAddLongColumn(col 7:bigint, col 11:bigint)(children: VectorUDFAdaptor(grouping(_col3, 1)) -> 7:bigint, VectorUDFAdaptor(grouping(_col3, 0)) -> 11:bigint) -> 12:bigint, IfExprColumnNull(col 7:boolean, col 3:string, null)(children: LongColEqualLongScalar(col 13:bigint, val 0)(children: LongColAddLongColumn(col 7:bigint, col 11:bigint)(children: VectorUDFAdaptor(grouping(_col3, 1)) -> 7:bigint, VectorUDFAdaptor(grouping(_col3, 0)) -> 11:bigint) -> 13:bigint) -> 7:boolean, col 3:string) -> 14:string + selectExpressions: LongColAddLongColumn(col 7:bigint, col 11:bigint)(children: GroupingColumn(col 5, mask 2) -> 7:bigint, GroupingColumn(col 5, mask 1) -> 11:bigint) -> 12:bigint, IfExprColumnNull(col 7:boolean, col 3:string, null)(children: LongColEqualLongScalar(col 13:bigint, val 0)(children: LongColAddLongColumn(col 7:bigint, col 11:bigint)(children: GroupingColumn(col 5, mask 2) -> 7:bigint, GroupingColumn(col 5, mask 1) -> 11:bigint) -> 13:bigint) -> 7:boolean, col 3:string) -> 14:string Statistics: Num rows: 1149975358 Data size: 101451159969 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col3 (type: bigint), _col5 (type: string), _col4 (type: int) diff --git ql/src/test/results/clientpositive/perf/tez/query86.q.out ql/src/test/results/clientpositive/perf/tez/query86.q.out index bbd76be..b04c644 100644 --- ql/src/test/results/clientpositive/perf/tez/query86.q.out +++ ql/src/test/results/clientpositive/perf/tez/query86.q.out @@ -272,7 +272,7 @@ STAGE PLANS: enabled: true enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true allNative: false - usesVectorUDFAdaptor: true + usesVectorUDFAdaptor: false vectorized: true Reduce Operator Tree: Group By Operator @@ -303,7 +303,7 @@ STAGE PLANS: Map-reduce partition columns: (grouping(_col3, 1) + grouping(_col3, 0)) (type: bigint), CASE WHEN ((grouping(_col3, 0) = 0)) THEN (_col0) ELSE (CAST( null AS STRING)) END (type: string) Reduce Sink Vectorization: className: VectorReduceSinkObjectHashOperator - keyExpressions: LongColAddLongColumn(col 4:bigint, col 5:bigint)(children: VectorUDFAdaptor(grouping(_col3, 1)) -> 4:bigint, VectorUDFAdaptor(grouping(_col3, 0)) -> 5:bigint) -> 6:bigint, IfExprColumnNull(col 5:boolean, col 0:string, null)(children: LongColEqualLongScalar(col 4:bigint, val 0)(children: VectorUDFAdaptor(grouping(_col3, 0)) -> 4:bigint) -> 5:boolean, col 0:string) -> 7:string + keyExpressions: LongColAddLongColumn(col 4:bigint, col 5:bigint)(children: GroupingColumn(col 2, mask 2) -> 4:bigint, GroupingColumn(col 2, mask 1) -> 5:bigint) -> 6:bigint, IfExprColumnNull(col 5:boolean, col 0:string, null)(children: LongColEqualLongScalar(col 4:bigint, val 0)(children: GroupingColumn(col 2, mask 1) -> 4:bigint) -> 5:boolean, col 0:string) -> 7:string native: true nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true Statistics: Num rows: 261364852 Data size: 35538061226 Basic stats: COMPLETE Column stats: NONE @@ -314,7 +314,7 @@ STAGE PLANS: enabled: true enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true allNative: true - usesVectorUDFAdaptor: true + usesVectorUDFAdaptor: false vectorized: true Reduce Operator Tree: Select Operator @@ -352,7 +352,7 @@ STAGE PLANS: functionNames: [rank] native: true orderExpressions: [col 2:decimal(17,2)] - partitionExpressions: [LongColAddLongColumn(col 7:bigint, col 8:bigint)(children: VectorUDFAdaptor(grouping(_col3, 1)) -> 7:bigint, VectorUDFAdaptor(grouping(_col3, 0)) -> 8:bigint) -> 9:bigint, IfExprColumnNull(col 8:boolean, col 3:string, null)(children: LongColEqualLongScalar(col 7:bigint, val 0)(children: VectorUDFAdaptor(grouping(_col3, 0)) -> 7:bigint) -> 8:boolean, col 3:string) -> 10:string] + partitionExpressions: [LongColAddLongColumn(col 7:bigint, col 8:bigint)(children: GroupingColumn(col 5, mask 2) -> 7:bigint, GroupingColumn(col 5, mask 1) -> 8:bigint) -> 9:bigint, IfExprColumnNull(col 8:boolean, col 3:string, null)(children: LongColEqualLongScalar(col 7:bigint, val 0)(children: GroupingColumn(col 5, mask 1) -> 7:bigint) -> 8:boolean, col 3:string) -> 10:string] Statistics: Num rows: 261364852 Data size: 35538061226 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: decimal(17,2)), _col0 (type: string), _col1 (type: string), (grouping(_col3, 1) + grouping(_col3, 0)) (type: bigint), rank_window_0 (type: int), CASE WHEN (((grouping(_col3, 1) + grouping(_col3, 0)) = 0)) THEN (_col0) ELSE (null) END (type: string) @@ -361,7 +361,7 @@ STAGE PLANS: className: VectorSelectOperator native: true projectedOutputColumnNums: [2, 3, 4, 12, 6, 14] - selectExpressions: LongColAddLongColumn(col 7:bigint, col 11:bigint)(children: VectorUDFAdaptor(grouping(_col3, 1)) -> 7:bigint, VectorUDFAdaptor(grouping(_col3, 0)) -> 11:bigint) -> 12:bigint, IfExprColumnNull(col 7:boolean, col 3:string, null)(children: LongColEqualLongScalar(col 13:bigint, val 0)(children: LongColAddLongColumn(col 7:bigint, col 11:bigint)(children: VectorUDFAdaptor(grouping(_col3, 1)) -> 7:bigint, VectorUDFAdaptor(grouping(_col3, 0)) -> 11:bigint) -> 13:bigint) -> 7:boolean, col 3:string) -> 14:string + selectExpressions: LongColAddLongColumn(col 7:bigint, col 11:bigint)(children: GroupingColumn(col 5, mask 2) -> 7:bigint, GroupingColumn(col 5, mask 1) -> 11:bigint) -> 12:bigint, IfExprColumnNull(col 7:boolean, col 3:string, null)(children: LongColEqualLongScalar(col 13:bigint, val 0)(children: LongColAddLongColumn(col 7:bigint, col 11:bigint)(children: GroupingColumn(col 5, mask 2) -> 7:bigint, GroupingColumn(col 5, mask 1) -> 11:bigint) -> 13:bigint) -> 7:boolean, col 3:string) -> 14:string Statistics: Num rows: 261364852 Data size: 35538061226 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col3 (type: bigint), _col5 (type: string), _col4 (type: int)