diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java index c3940cbfe3..7dc4c812c7 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java @@ -94,7 +94,6 @@ import org.apache.hadoop.hive.ql.exec.vector.udf.VectorUDFAdaptor; import org.apache.hadoop.hive.ql.exec.vector.udf.VectorUDFArgDesc; import org.apache.hadoop.hive.ql.metadata.HiveException; -import org.apache.hadoop.hive.ql.optimizer.physical.Vectorizer; import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.plan.AggregationDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc; @@ -102,7 +101,6 @@ import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeDynamicValueDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc; -import org.apache.hadoop.hive.ql.plan.GroupByDesc; import org.apache.hadoop.hive.ql.udf.*; import org.apache.hadoop.hive.ql.udf.generic.*; import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator.Mode; @@ -126,8 +124,6 @@ import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.Text; -import org.apache.hadoop.util.StringUtils; -import org.apache.hive.common.util.DateUtils; import com.google.common.annotations.VisibleForTesting; @@ -2322,13 +2318,10 @@ private VectorExpression getBetweenFilterExpression(List childExpr return createVectorExpression(cl, childrenAfterNot, VectorExpressionDescriptor.Mode.PROJECTION, returnType); } - private boolean isColumnOrNonNullConst(ExprNodeDesc exprNodeDesc) { - if (exprNodeDesc instanceof ExprNodeColumnDesc) { - return true; - } + private boolean isNullConst(ExprNodeDesc exprNodeDesc) { if (exprNodeDesc instanceof ExprNodeConstantDesc) { String typeString = exprNodeDesc.getTypeString(); - if (!typeString.equalsIgnoreCase("void")) { + if (typeString.equalsIgnoreCase("void")) { return true; } } @@ -2341,33 +2334,47 @@ private VectorExpression getWhenExpression(List childExpr, if (mode != VectorExpressionDescriptor.Mode.PROJECTION) { return null; } - if (childExpr.size() != 3) { - // For now, we only optimize the 2 value case. - return null; - } + final int size = childExpr.size(); - /* - * When we have 2 simple values: - * CASE WHEN boolExpr THEN column | const ELSE column | const END - * then we can convert to: IF (boolExpr THEN column | const ELSE column | const) - */ - // CONSIDER: Adding a version of IfExpr* than can handle a non-column/const expression in the - // THEN or ELSE. - ExprNodeDesc exprNodeDesc1 = childExpr.get(1); - ExprNodeDesc exprNodeDesc2 = childExpr.get(2); - if (isColumnOrNonNullConst(exprNodeDesc1) && - isColumnOrNonNullConst(exprNodeDesc2)) { - // Yes. - GenericUDFIf genericUDFIf = new GenericUDFIf(); - return - getVectorExpressionForUdf( - genericUDFIf, - GenericUDFIf.class, - childExpr, - mode, - returnType); - } - return null; // Not handled by vector classes yet. + final ExprNodeDesc whenDesc = childExpr.get(0); + final ExprNodeDesc thenDesc = childExpr.get(1); + final ExprNodeDesc elseDesc; + + if (size == 2) { + elseDesc = new ExprNodeConstantDesc(returnType, null); + } else if (size == 3) { + elseDesc = childExpr.get(2); + } else { + final GenericUDFWhen udfWhen = new GenericUDFWhen(); + elseDesc = new ExprNodeGenericFuncDesc(returnType, udfWhen, udfWhen.getUdfName(), + childExpr.subList(2, childExpr.size())); + } + + if (isNullConst(thenDesc)) { + final VectorExpression whenExpr = getVectorExpression(whenDesc, mode); + final VectorExpression elseExpr = getVectorExpression(elseDesc, mode); + final VectorExpression resultExpr = new IfExprNullColumn( + whenExpr.getOutputColumn(), elseExpr.getOutputColumn(), + ocm.allocateOutputColumn(returnType)); + resultExpr.setChildExpressions(new VectorExpression[] {whenExpr, elseExpr}); + resultExpr.setOutputType(returnType.getTypeName()); + return resultExpr; + } + if (isNullConst(elseDesc)) { + final VectorExpression whenExpr = getVectorExpression(whenDesc, mode); + final VectorExpression thenExpr = getVectorExpression(thenDesc, mode); + final VectorExpression resultExpr = new IfExprColumnNull( + whenExpr.getOutputColumn(), thenExpr.getOutputColumn(), + ocm.allocateOutputColumn(returnType)); + resultExpr.setChildExpressions(new VectorExpression[] {whenExpr, thenExpr}); + resultExpr.setOutputType(returnType.getTypeName()); + return resultExpr; + } + final GenericUDFIf genericUDFIf = new GenericUDFIf(); + final List ifChildExpr = Arrays.asList(whenDesc, thenDesc, elseDesc); + final ExprNodeGenericFuncDesc exprNodeDesc = + new ExprNodeGenericFuncDesc(returnType, genericUDFIf, "if", ifChildExpr); + return getVectorExpression(exprNodeDesc, mode); } /* diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprColumnNull.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprColumnNull.java new file mode 100644 index 0000000000..8cae274d28 --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprColumnNull.java @@ -0,0 +1,109 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.exec.vector.expressions; + +import org.apache.hadoop.hive.ql.exec.vector.ColumnVector; +import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector; +import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; +import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; + +public class IfExprColumnNull extends VectorExpression { + + private static final long serialVersionUID = 1L; + + private final int arg1Column; + private final int arg2Column; + private final int outputColumn; + + public IfExprColumnNull(int arg1Column, int arg2Column, int outputColumn) { + this.arg1Column = arg1Column; + this.arg2Column = arg2Column; + this.outputColumn = outputColumn; + } + + @Override + public void evaluate(VectorizedRowBatch batch) { + + if (childExpressions != null) { + super.evaluateChildren(batch); + } + + final LongColumnVector arg1ColVector = (LongColumnVector) batch.cols[arg1Column]; + final ColumnVector arg2ColVector = batch.cols[arg2Column]; + final ColumnVector outputColVector = batch.cols[outputColumn]; + + final int[] sel = batch.selected; + final int n = batch.size; + final boolean[] null1 = arg1ColVector.isNull; + final long[] vector1 = arg1ColVector.vector; + final boolean[] isNull = outputColVector.isNull; + + if (n == 0) { + return; + } + + arg2ColVector.flatten(batch.selectedInUse, sel, n); + + if (arg1ColVector.isRepeating) { + if (!null1[0] && vector1[0] == 1) { + outputColVector.setElement(0, 0, arg2ColVector); + } else { + outputColVector.noNulls = false; + isNull[0] = true; + } + return; + } + if (batch.selectedInUse) { + for (int j = 0; j < n; j++) { + int i = sel[j]; + if (!null1[0] && vector1[i] == 1) { + outputColVector.setElement(i, i, arg2ColVector); + } else { + outputColVector.noNulls = false; + isNull[i] = true; + } + } + } else { + for (int i = 0; i < n; i++) { + if (!null1[0] && vector1[i] == 1) { + outputColVector.setElement(i, i, arg2ColVector); + } else { + outputColVector.noNulls = false; + isNull[i] = true; + } + } + } + + arg2ColVector.unFlatten(); + } + + @Override + public int getOutputColumn() { + return outputColumn; + } + + @Override + public String vectorExpressionParameters() { + return "col " + arg1Column + ", col "+ arg2Column + ", null"; + } + + @Override + public VectorExpressionDescriptor.Descriptor getDescriptor() { + throw new UnsupportedOperationException("Undefined descriptor"); + } +} diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprNullColumn.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprNullColumn.java new file mode 100644 index 0000000000..156fcc45e1 --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IfExprNullColumn.java @@ -0,0 +1,109 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.exec.vector.expressions; + +import org.apache.hadoop.hive.ql.exec.vector.ColumnVector; +import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector; +import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; +import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; + +public class IfExprNullColumn extends VectorExpression { + + private static final long serialVersionUID = 1L; + + private final int arg1Column; + private final int arg2Column; + private final int outputColumn; + + public IfExprNullColumn(int arg1Column, int arg2Column, int outputColumn) { + this.arg1Column = arg1Column; + this.arg2Column = arg2Column; + this.outputColumn = outputColumn; + } + + @Override + public void evaluate(VectorizedRowBatch batch) { + + if (childExpressions != null) { + super.evaluateChildren(batch); + } + + final LongColumnVector arg1ColVector = (LongColumnVector) batch.cols[arg1Column]; + final ColumnVector arg2ColVector = batch.cols[arg2Column]; + final ColumnVector outputColVector = batch.cols[outputColumn]; + + final int[] sel = batch.selected; + final int n = batch.size; + final boolean[] null1 = arg1ColVector.isNull; + final long[] vector1 = arg1ColVector.vector; + final boolean[] isNull = outputColVector.isNull; + + if (n == 0) { + return; + } + + arg2ColVector.flatten(batch.selectedInUse, sel, n); + + if (arg1ColVector.isRepeating) { + if (!null1[0] && vector1[0] == 1) { + outputColVector.noNulls = false; + isNull[0] = true; + } else { + outputColVector.setElement(0, 0, arg2ColVector); + } + return; + } + if (batch.selectedInUse) { + for (int j = 0; j < n; j++) { + int i = sel[j]; + if (!null1[0] && vector1[i] == 1) { + outputColVector.noNulls = false; + isNull[i] = true; + } else { + outputColVector.setElement(i, i, arg2ColVector); + } + } + } else { + for (int i = 0; i < n; i++) { + if (!null1[0] && vector1[i] == 1) { + outputColVector.noNulls = false; + isNull[i] = true; + } else { + outputColVector.setElement(i, i, arg2ColVector); + } + } + } + + arg2ColVector.unFlatten(); + } + + @Override + public int getOutputColumn() { + return outputColumn; + } + + @Override + public String vectorExpressionParameters() { + return "col " + arg1Column + ", null, col "+ arg2Column; + } + + @Override + public VectorExpressionDescriptor.Descriptor getDescriptor() { + throw new UnsupportedOperationException("Undefined descriptor"); + } +} diff --git ql/src/test/results/clientpositive/llap/vector_adaptor_usage_mode.q.out ql/src/test/results/clientpositive/llap/vector_adaptor_usage_mode.q.out index a2ce36564e..6a0f490cd4 100644 --- ql/src/test/results/clientpositive/llap/vector_adaptor_usage_mode.q.out +++ ql/src/test/results/clientpositive/llap/vector_adaptor_usage_mode.q.out @@ -965,12 +965,27 @@ STAGE PLANS: TableScan alias: count_case_groupby Statistics: Num rows: 5 Data size: 452 Basic stats: COMPLETE Column stats: NONE + TableScan Vectorization: + native: true + projectedOutputColumns: [0, 1] Select Operator expressions: key (type: string), CASE WHEN (bool) THEN (1) WHEN ((not bool)) THEN (0) ELSE (null) END (type: int) outputColumnNames: _col0, _col1 + Select Vectorization: + className: VectorSelectOperator + native: true + projectedOutputColumns: [0, 5] + selectExpressions: IfExprLongScalarLongColumn(col 1, val 1, col 4)(children: IfExprColumnNull(col 2, col 3, null)(children: NotCol(col 1) -> 2:boolean, ConstantVectorExpression(val 0) -> 3:long) -> 4:int) -> 5:long Statistics: Num rows: 5 Data size: 452 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(_col1) + Group By Vectorization: + aggregators: VectorUDAFCount(col 5) -> bigint + className: VectorGroupByOperator + vectorOutput: true + keyExpressions: col 0 + native: false + projectedOutputColumns: [0] keys: _col0 (type: string) mode: hash outputColumnNames: _col0, _col1 @@ -979,16 +994,22 @@ STAGE PLANS: key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) + Reduce Sink Vectorization: + className: VectorReduceSinkStringOperator + native: true + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true Statistics: Num rows: 5 Data size: 452 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) - Execution mode: llap + Execution mode: vectorized, llap LLAP IO: all inputs Map Vectorization: enabled: true enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true + groupByVectorOutput: true inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - notVectorizedReason: Select expression for SELECT operator: Could not vectorize expression (mode = PROJECTION): GenericUDFWhen(Column[bool], Const int 1, GenericUDFOPNot(Column[bool]), Const int 0, Const void null) because hive.vectorized.adaptor.usage.mode=none - vectorized: false + allNative: false + usesVectorUDFAdaptor: false + vectorized: true Reducer 2 Execution mode: vectorized, llap Reduce Vectorization: @@ -1078,13 +1099,13 @@ STAGE PLANS: Select Vectorization: className: VectorSelectOperator native: true - projectedOutputColumns: [0, 3] - selectExpressions: VectorUDFAdaptor(CASE WHEN (bool) THEN (1) WHEN ((not bool)) THEN (0) ELSE (null) END)(children: NotCol(col 1) -> 2:boolean) -> 3:int + projectedOutputColumns: [0, 5] + selectExpressions: IfExprLongScalarLongColumn(col 1, val 1, col 4)(children: IfExprColumnNull(col 2, col 3, null)(children: NotCol(col 1) -> 2:boolean, ConstantVectorExpression(val 0) -> 3:long) -> 4:int) -> 5:long Statistics: Num rows: 5 Data size: 452 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(_col1) Group By Vectorization: - aggregators: VectorUDAFCount(col 3) -> bigint + aggregators: VectorUDAFCount(col 5) -> bigint className: VectorGroupByOperator vectorOutput: true keyExpressions: col 0 @@ -1112,7 +1133,7 @@ STAGE PLANS: groupByVectorOutput: true inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat allNative: false - usesVectorUDFAdaptor: true + usesVectorUDFAdaptor: false vectorized: true Reducer 2 Execution mode: vectorized, llap diff --git ql/src/test/results/clientpositive/llap/vector_groupby_grouping_sets_grouping.q.out ql/src/test/results/clientpositive/llap/vector_groupby_grouping_sets_grouping.q.out index 98cd4d1d75..fccb6d6f54 100644 --- ql/src/test/results/clientpositive/llap/vector_groupby_grouping_sets_grouping.q.out +++ ql/src/test/results/clientpositive/llap/vector_groupby_grouping_sets_grouping.q.out @@ -772,7 +772,7 @@ STAGE PLANS: Execution mode: vectorized, llap LLAP IO: all inputs Reducer 2 - Execution mode: vectorized, llap + Execution mode: llap Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: int), KEY._col1 (type: int), KEY._col2 (type: int) diff --git ql/src/test/results/clientpositive/llap/vector_ptf_part_simple.q.out ql/src/test/results/clientpositive/llap/vector_ptf_part_simple.q.out index c9c7d8cf15..c2f5a29271 100644 --- ql/src/test/results/clientpositive/llap/vector_ptf_part_simple.q.out +++ ql/src/test/results/clientpositive/llap/vector_ptf_part_simple.q.out @@ -2773,8 +2773,8 @@ STAGE PLANS: Map-reduce partition columns: p_mfgr (type: string), CASE WHEN ((p_mfgr = 'Manufacturer#2')) THEN (2000-01-01 00:00:00.0) ELSE (null) END (type: timestamp) Reduce Sink Vectorization: className: VectorReduceSinkMultiKeyOperator - keyColumns: [0, 4] - keyExpressions: VectorUDFAdaptor(CASE WHEN ((p_mfgr = 'Manufacturer#2')) THEN (2000-01-01 00:00:00.0) ELSE (null) END)(children: StringGroupColEqualStringScalar(col 0, val Manufacturer#2) -> 3:boolean) -> 4:timestamp + keyColumns: [0, 5] + keyExpressions: IfExprColumnNull(col 3, col 4, null)(children: StringGroupColEqualStringScalar(col 0, val Manufacturer#2) -> 3:boolean, ConstantVectorExpression(val 2000-01-01 00:00:00.0) -> 4:timestamp) -> 5:timestamp native: true nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true valueColumns: [1, 2] @@ -2788,14 +2788,14 @@ STAGE PLANS: groupByVectorOutput: true inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat allNative: true - usesVectorUDFAdaptor: true + usesVectorUDFAdaptor: false vectorized: true rowBatchContext: dataColumnCount: 3 includeColumns: [0, 1, 2] dataColumns: p_mfgr:string, p_name:string, p_retailprice:double partitionColumnCount: 0 - scratchColumnTypeNames: bigint, timestamp + scratchColumnTypeNames: bigint, timestamp, timestamp Reducer 2 Execution mode: llap Reduce Vectorization: @@ -2941,11 +2941,11 @@ STAGE PLANS: Map-reduce partition columns: p_mfgr (type: string), CASE WHEN ((p_mfgr = 'Manufacturer#2')) THEN (2000-01-01 00:00:00.0) ELSE (null) END (type: timestamp) Reduce Sink Vectorization: className: VectorReduceSinkObjectHashOperator - keyColumns: [0, 4, 1] - keyExpressions: VectorUDFAdaptor(CASE WHEN ((p_mfgr = 'Manufacturer#2')) THEN (2000-01-01 00:00:00.0) ELSE (null) END)(children: StringGroupColEqualStringScalar(col 0, val Manufacturer#2) -> 3:boolean) -> 4:timestamp + keyColumns: [0, 5, 1] + keyExpressions: IfExprColumnNull(col 3, col 4, null)(children: StringGroupColEqualStringScalar(col 0, val Manufacturer#2) -> 3:boolean, ConstantVectorExpression(val 2000-01-01 00:00:00.0) -> 4:timestamp) -> 5:timestamp native: true nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - partitionColumns: [0, 5] + partitionColumns: [0, 8] valueColumns: [2] Statistics: Num rows: 40 Data size: 9048 Basic stats: COMPLETE Column stats: NONE value expressions: p_retailprice (type: double) @@ -2957,14 +2957,14 @@ STAGE PLANS: groupByVectorOutput: true inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat allNative: true - usesVectorUDFAdaptor: true + usesVectorUDFAdaptor: false vectorized: true rowBatchContext: dataColumnCount: 3 includeColumns: [0, 1, 2] dataColumns: p_mfgr:string, p_name:string, p_retailprice:double partitionColumnCount: 0 - scratchColumnTypeNames: bigint, timestamp, timestamp + scratchColumnTypeNames: bigint, timestamp, timestamp, bigint, timestamp, timestamp Reducer 2 Execution mode: llap Reduce Vectorization: diff --git ql/src/test/results/clientpositive/llap/vector_when_case_null.q.out ql/src/test/results/clientpositive/llap/vector_when_case_null.q.out index 06dde804dc..28edb6f5b3 100644 --- ql/src/test/results/clientpositive/llap/vector_when_case_null.q.out +++ ql/src/test/results/clientpositive/llap/vector_when_case_null.q.out @@ -50,13 +50,13 @@ STAGE PLANS: Select Vectorization: className: VectorSelectOperator native: true - projectedOutputColumns: [0, 3] - selectExpressions: VectorUDFAdaptor(CASE WHEN (bool) THEN (1) WHEN ((not bool)) THEN (0) ELSE (null) END)(children: NotCol(col 1) -> 2:boolean) -> 3:int + projectedOutputColumns: [0, 5] + selectExpressions: IfExprLongScalarLongColumn(col 1, val 1, col 4)(children: IfExprColumnNull(col 2, col 3, null)(children: NotCol(col 1) -> 2:boolean, ConstantVectorExpression(val 0) -> 3:long) -> 4:int) -> 5:long Statistics: Num rows: 5 Data size: 452 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(_col1) Group By Vectorization: - aggregators: VectorUDAFCount(col 3) -> bigint + aggregators: VectorUDAFCount(col 5) -> bigint className: VectorGroupByOperator vectorOutput: true keyExpressions: col 0 @@ -84,7 +84,7 @@ STAGE PLANS: groupByVectorOutput: true inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat allNative: false - usesVectorUDFAdaptor: true + usesVectorUDFAdaptor: false vectorized: true Reducer 2 Execution mode: vectorized, llap diff --git ql/src/test/results/clientpositive/llap/vectorized_case.q.out ql/src/test/results/clientpositive/llap/vectorized_case.q.out index beda526747..0beaba82b7 100644 --- ql/src/test/results/clientpositive/llap/vectorized_case.q.out +++ ql/src/test/results/clientpositive/llap/vectorized_case.q.out @@ -68,8 +68,8 @@ STAGE PLANS: Select Vectorization: className: VectorSelectOperator native: true - projectedOutputColumns: [1, 14, 15] - selectExpressions: VectorUDFAdaptor(CASE WHEN ((csmallint = 418)) THEN ('a') WHEN ((csmallint = 12205)) THEN ('b') ELSE ('c') END)(children: LongColEqualLongScalar(col 1, val 418) -> 12:long, LongColEqualLongScalar(col 1, val 12205) -> 13:long) -> 14:string, VectorUDFAdaptor(CASE WHEN ((csmallint = 418)) THEN ('a') WHEN ((csmallint = 12205)) THEN ('b') ELSE ('c') END)(children: LongColEqualLongScalar(col 1, val 418) -> 12:long, LongColEqualLongScalar(col 1, val 12205) -> 13:long) -> 15:string + projectedOutputColumns: [1, 15, 16] + selectExpressions: IfExprStringScalarStringGroupColumn(col 12, val a, col 14)(children: LongColEqualLongScalar(col 1, val 418) -> 12:long, IfExprStringScalarStringScalar(col 13, val b, val c)(children: LongColEqualLongScalar(col 1, val 12205) -> 13:long) -> 14:String) -> 15:String, IfExprStringScalarStringGroupColumn(col 12, val a, col 14)(children: LongColEqualLongScalar(col 1, val 418) -> 12:long, IfExprStringScalarStringScalar(col 13, val b, val c)(children: LongColEqualLongScalar(col 1, val 12205) -> 13:long) -> 14:String) -> 16:String Statistics: Num rows: 6 Data size: 2228 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false @@ -89,7 +89,7 @@ STAGE PLANS: groupByVectorOutput: true inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat allNative: false - usesVectorUDFAdaptor: true + usesVectorUDFAdaptor: false vectorized: true Stage: Stage-0 @@ -210,8 +210,8 @@ STAGE PLANS: Select Vectorization: className: VectorSelectOperator native: true - projectedOutputColumns: [1, 14, 15] - selectExpressions: VectorUDFAdaptor(CASE WHEN ((csmallint = 418)) THEN ('a') WHEN ((csmallint = 12205)) THEN ('b') ELSE (null) END)(children: LongColEqualLongScalar(col 1, val 418) -> 12:long, LongColEqualLongScalar(col 1, val 12205) -> 13:long) -> 14:string, VectorUDFAdaptor(CASE WHEN ((csmallint = 418)) THEN ('a') WHEN ((csmallint = 12205)) THEN (null) ELSE ('c') END)(children: LongColEqualLongScalar(col 1, val 418) -> 12:long, LongColEqualLongScalar(col 1, val 12205) -> 13:long) -> 15:string + projectedOutputColumns: [1, 16, 19] + selectExpressions: IfExprStringScalarStringGroupColumn(col 12, val a, col 15)(children: LongColEqualLongScalar(col 1, val 418) -> 12:long, IfExprColumnNull(col 13, col 14, null)(children: LongColEqualLongScalar(col 1, val 12205) -> 13:long, ConstantVectorExpression(val b) -> 14:string) -> 15:string) -> 16:String, IfExprStringScalarStringGroupColumn(col 12, val a, col 18)(children: LongColEqualLongScalar(col 1, val 418) -> 12:long, IfExprNullColumn(col 17, null, col 15)(children: LongColEqualLongScalar(col 1, val 12205) -> 17:long, ConstantVectorExpression(val c) -> 15:string) -> 18:string) -> 19:String Statistics: Num rows: 6 Data size: 2228 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false @@ -231,7 +231,7 @@ STAGE PLANS: groupByVectorOutput: true inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat allNative: false - usesVectorUDFAdaptor: true + usesVectorUDFAdaptor: false vectorized: true Stage: Stage-0 diff --git ql/src/test/results/clientpositive/spark/vectorized_case.q.out ql/src/test/results/clientpositive/spark/vectorized_case.q.out index 133842d660..ffac02c80d 100644 --- ql/src/test/results/clientpositive/spark/vectorized_case.q.out +++ ql/src/test/results/clientpositive/spark/vectorized_case.q.out @@ -68,8 +68,8 @@ STAGE PLANS: Select Vectorization: className: VectorSelectOperator native: true - projectedOutputColumns: [1, 14, 15] - selectExpressions: VectorUDFAdaptor(CASE WHEN ((csmallint = 418)) THEN ('a') WHEN ((csmallint = 12205)) THEN ('b') ELSE ('c') END)(children: LongColEqualLongScalar(col 1, val 418) -> 12:long, LongColEqualLongScalar(col 1, val 12205) -> 13:long) -> 14:string, VectorUDFAdaptor(CASE WHEN ((csmallint = 418)) THEN ('a') WHEN ((csmallint = 12205)) THEN ('b') ELSE ('c') END)(children: LongColEqualLongScalar(col 1, val 418) -> 12:long, LongColEqualLongScalar(col 1, val 12205) -> 13:long) -> 15:string + projectedOutputColumns: [1, 15, 16] + selectExpressions: IfExprStringScalarStringGroupColumn(col 12, val a, col 14)(children: LongColEqualLongScalar(col 1, val 418) -> 12:long, IfExprStringScalarStringScalar(col 13, val b, val c)(children: LongColEqualLongScalar(col 1, val 12205) -> 13:long) -> 14:String) -> 15:String, IfExprStringScalarStringGroupColumn(col 12, val a, col 14)(children: LongColEqualLongScalar(col 1, val 418) -> 12:long, IfExprStringScalarStringScalar(col 13, val b, val c)(children: LongColEqualLongScalar(col 1, val 12205) -> 13:long) -> 14:String) -> 16:String Statistics: Num rows: 12288 Data size: 377237 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -88,7 +88,7 @@ STAGE PLANS: groupByVectorOutput: true inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat allNative: false - usesVectorUDFAdaptor: true + usesVectorUDFAdaptor: false vectorized: true Stage: Stage-0 @@ -209,8 +209,8 @@ STAGE PLANS: Select Vectorization: className: VectorSelectOperator native: true - projectedOutputColumns: [1, 14, 15] - selectExpressions: VectorUDFAdaptor(CASE WHEN ((csmallint = 418)) THEN ('a') WHEN ((csmallint = 12205)) THEN ('b') ELSE (null) END)(children: LongColEqualLongScalar(col 1, val 418) -> 12:long, LongColEqualLongScalar(col 1, val 12205) -> 13:long) -> 14:string, VectorUDFAdaptor(CASE WHEN ((csmallint = 418)) THEN ('a') WHEN ((csmallint = 12205)) THEN (null) ELSE ('c') END)(children: LongColEqualLongScalar(col 1, val 418) -> 12:long, LongColEqualLongScalar(col 1, val 12205) -> 13:long) -> 15:string + projectedOutputColumns: [1, 16, 19] + selectExpressions: IfExprStringScalarStringGroupColumn(col 12, val a, col 15)(children: LongColEqualLongScalar(col 1, val 418) -> 12:long, IfExprColumnNull(col 13, col 14, null)(children: LongColEqualLongScalar(col 1, val 12205) -> 13:long, ConstantVectorExpression(val b) -> 14:string) -> 15:string) -> 16:String, IfExprStringScalarStringGroupColumn(col 12, val a, col 18)(children: LongColEqualLongScalar(col 1, val 418) -> 12:long, IfExprNullColumn(col 17, null, col 15)(children: LongColEqualLongScalar(col 1, val 12205) -> 17:long, ConstantVectorExpression(val c) -> 15:string) -> 18:string) -> 19:String Statistics: Num rows: 12288 Data size: 377237 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -229,7 +229,7 @@ STAGE PLANS: groupByVectorOutput: true inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat allNative: false - usesVectorUDFAdaptor: true + usesVectorUDFAdaptor: false vectorized: true Stage: Stage-0 diff --git ql/src/test/results/clientpositive/vector_when_case_null.q.out ql/src/test/results/clientpositive/vector_when_case_null.q.out index a7ab4ef43e..5ae4b99b55 100644 --- ql/src/test/results/clientpositive/vector_when_case_null.q.out +++ ql/src/test/results/clientpositive/vector_when_case_null.q.out @@ -44,13 +44,13 @@ STAGE PLANS: Select Vectorization: className: VectorSelectOperator native: true - projectedOutputColumns: [0, 3] - selectExpressions: VectorUDFAdaptor(CASE WHEN (bool) THEN (1) WHEN ((not bool)) THEN (0) ELSE (null) END)(children: NotCol(col 1) -> 2:boolean) -> 3:int + projectedOutputColumns: [0, 5] + selectExpressions: IfExprLongScalarLongColumn(col 1, val 1, col 4)(children: IfExprColumnNull(col 2, col 3, null)(children: NotCol(col 1) -> 2:boolean, ConstantVectorExpression(val 0) -> 3:long) -> 4:int) -> 5:long Statistics: Num rows: 5 Data size: 452 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(_col1) Group By Vectorization: - aggregators: VectorUDAFCount(col 3) -> bigint + aggregators: VectorUDAFCount(col 5) -> bigint className: VectorGroupByOperator vectorOutput: true keyExpressions: col 0 @@ -78,7 +78,7 @@ STAGE PLANS: groupByVectorOutput: true inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat allNative: false - usesVectorUDFAdaptor: true + usesVectorUDFAdaptor: false vectorized: true Reduce Vectorization: enabled: false diff --git ql/src/test/results/clientpositive/vectorized_case.q.out ql/src/test/results/clientpositive/vectorized_case.q.out index 5a7a8a2cbf..b1b5e54fa3 100644 --- ql/src/test/results/clientpositive/vectorized_case.q.out +++ ql/src/test/results/clientpositive/vectorized_case.q.out @@ -65,8 +65,8 @@ STAGE PLANS: Select Vectorization: className: VectorSelectOperator native: true - projectedOutputColumns: [1, 14, 15] - selectExpressions: VectorUDFAdaptor(CASE WHEN ((csmallint = 418)) THEN ('a') WHEN ((csmallint = 12205)) THEN ('b') ELSE ('c') END)(children: LongColEqualLongScalar(col 1, val 418) -> 12:long, LongColEqualLongScalar(col 1, val 12205) -> 13:long) -> 14:string, VectorUDFAdaptor(CASE WHEN ((csmallint = 418)) THEN ('a') WHEN ((csmallint = 12205)) THEN ('b') ELSE ('c') END)(children: LongColEqualLongScalar(col 1, val 418) -> 12:long, LongColEqualLongScalar(col 1, val 12205) -> 13:long) -> 15:string + projectedOutputColumns: [1, 15, 16] + selectExpressions: IfExprStringScalarStringGroupColumn(col 12, val a, col 14)(children: LongColEqualLongScalar(col 1, val 418) -> 12:long, IfExprStringScalarStringScalar(col 13, val b, val c)(children: LongColEqualLongScalar(col 1, val 12205) -> 13:long) -> 14:String) -> 15:String, IfExprStringScalarStringGroupColumn(col 12, val a, col 14)(children: LongColEqualLongScalar(col 1, val 418) -> 12:long, IfExprStringScalarStringScalar(col 13, val b, val c)(children: LongColEqualLongScalar(col 1, val 12205) -> 13:long) -> 14:String) -> 16:String Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -85,7 +85,7 @@ STAGE PLANS: groupByVectorOutput: true inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat allNative: false - usesVectorUDFAdaptor: true + usesVectorUDFAdaptor: false vectorized: true Stage: Stage-0 @@ -203,8 +203,8 @@ STAGE PLANS: Select Vectorization: className: VectorSelectOperator native: true - projectedOutputColumns: [1, 14, 15] - selectExpressions: VectorUDFAdaptor(CASE WHEN ((csmallint = 418)) THEN ('a') WHEN ((csmallint = 12205)) THEN ('b') ELSE (null) END)(children: LongColEqualLongScalar(col 1, val 418) -> 12:long, LongColEqualLongScalar(col 1, val 12205) -> 13:long) -> 14:string, VectorUDFAdaptor(CASE WHEN ((csmallint = 418)) THEN ('a') WHEN ((csmallint = 12205)) THEN (null) ELSE ('c') END)(children: LongColEqualLongScalar(col 1, val 418) -> 12:long, LongColEqualLongScalar(col 1, val 12205) -> 13:long) -> 15:string + projectedOutputColumns: [1, 16, 19] + selectExpressions: IfExprStringScalarStringGroupColumn(col 12, val a, col 15)(children: LongColEqualLongScalar(col 1, val 418) -> 12:long, IfExprColumnNull(col 13, col 14, null)(children: LongColEqualLongScalar(col 1, val 12205) -> 13:long, ConstantVectorExpression(val b) -> 14:string) -> 15:string) -> 16:String, IfExprStringScalarStringGroupColumn(col 12, val a, col 18)(children: LongColEqualLongScalar(col 1, val 418) -> 12:long, IfExprNullColumn(col 17, null, col 15)(children: LongColEqualLongScalar(col 1, val 12205) -> 17:long, ConstantVectorExpression(val c) -> 15:string) -> 18:string) -> 19:String Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -223,7 +223,7 @@ STAGE PLANS: groupByVectorOutput: true inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat allNative: false - usesVectorUDFAdaptor: true + usesVectorUDFAdaptor: false vectorized: true Stage: Stage-0