commit 188d0be48ea463532bc2a9b1b63b9da7c8a04aab Author: Vihang Karajgaonkar Date: Wed Nov 15 14:23:27 2017 -0800 HIVE-16756 : Vectorization: LongColModuloLongColumn throws java.lang.ArithmeticException: / by zero diff --git a/ql/src/gen/vectorization/ExpressionTemplates/ColumnDivideColumn.txt b/ql/src/gen/vectorization/ExpressionTemplates/ColumnDivideColumn.txt index 8b586b1f00ce7d6081f973a5736100d8941f79bc..843bb5ea638bf8cf23fab8724fd8af676a5be03a 100644 --- a/ql/src/gen/vectorization/ExpressionTemplates/ColumnDivideColumn.txt +++ b/ql/src/gen/vectorization/ExpressionTemplates/ColumnDivideColumn.txt @@ -25,7 +25,7 @@ import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; /** - * Generated from template ColumnArithmeticColumn.txt, which covers binary arithmetic + * Generated from template ColumnDivideColumn.txt, which covers division and modulo * expressions between columns. */ public class extends VectorExpression { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/LongColModuloLongColumn.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/LongColModuloLongColumn.java new file mode 100644 index 0000000000000000000000000000000000000000..19fc3a615216555a2978c0210122b53497dec4f2 --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/LongColModuloLongColumn.java @@ -0,0 +1,183 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.exec.vector.expressions; + +import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; +import org.apache.hadoop.hive.ql.exec.vector.expressions.NullUtil; +import org.apache.hadoop.hive.ql.exec.vector.*; +import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; +import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; + +/** + * This operation is handled as a special case because Hive + * long%long division returns needs special handling to avoid + * for divide by zero exception + */ +public class LongColModuloLongColumn extends VectorExpression { + + private static final long serialVersionUID = 1L; + + private final int colNum1; + private final int colNum2; + + public LongColModuloLongColumn(int colNum1, int colNum2, int outputColumnNum) { + super(outputColumnNum); + this.colNum1 = colNum1; + this.colNum2 = colNum2; + } + + public LongColModuloLongColumn() { + super(); + + // Dummy final assignments. + colNum1 = -1; + colNum2 = -1; + } + + @Override + public void evaluate(VectorizedRowBatch batch) { + + if (childExpressions != null) { + super.evaluateChildren(batch); + } + + LongColumnVector inputColVector1 = (LongColumnVector) batch.cols[colNum1]; + LongColumnVector inputColVector2 = (LongColumnVector) batch.cols[colNum2]; + LongColumnVector outputColVector = (LongColumnVector) batch.cols[outputColumnNum]; + int[] sel = batch.selected; + int n = batch.size; + long[] vector1 = inputColVector1.vector; + long[] vector2 = inputColVector2.vector; + long[] outputVector = outputColVector.vector; + + // return immediately if batch is empty + if (n == 0) { + return; + } + + outputColVector.isRepeating = + inputColVector1.isRepeating && inputColVector2.isRepeating + || inputColVector1.isRepeating && !inputColVector1.noNulls && inputColVector1.isNull[0] + || inputColVector2.isRepeating && !inputColVector2.noNulls && inputColVector2.isNull[0]; + + // Handle nulls first + NullUtil.propagateNullsColCol( + inputColVector1, inputColVector2, outputColVector, sel, n, batch.selectedInUse); + + /* Disregard nulls for processing. In other words, + * the arithmetic operation is performed even if one or + * more inputs are null. This is to improve speed by avoiding + * conditional checks in the inner loop. + */ + boolean hasDivBy0 = false; + if (inputColVector1.isRepeating && inputColVector2.isRepeating) { + long denom = vector2[0]; + hasDivBy0 = hasDivBy0 || (denom == 0); + if (denom != 0) { + outputVector[0] = vector1[0] % denom; + } + } else if (inputColVector1.isRepeating) { + final long vector1Value = vector1[0]; + if (batch.selectedInUse) { + for(int j = 0; j != n; j++) { + int i = sel[j]; + long denom = vector2[i]; + hasDivBy0 = hasDivBy0 || (denom == 0); + if (denom != 0) { + outputVector[i] = vector1Value % denom; + } + } + } else { + for(int i = 0; i != n; i++) { + hasDivBy0 = hasDivBy0 || (vector2[i] == 0); + if (vector2[i] != 0) { + outputVector[i] = vector1Value % vector2[i]; + } + } + } + } else if (inputColVector2.isRepeating) { + final long vector2Value = vector2[0]; + if (vector2Value == 0) { + // Denominator is zero, convert the batch to nulls + outputColVector.noNulls = false; + outputColVector.isRepeating = true; + outputColVector.isNull[0] = true; + } else if (batch.selectedInUse) { + for(int j = 0; j != n; j++) { + int i = sel[j]; + outputVector[i] = vector1[i] % vector2Value; + } + } else { + for(int i = 0; i != n; i++) { + outputVector[i] = vector1[i] % vector2Value; + } + } + } else { + if (batch.selectedInUse) { + for(int j = 0; j != n; j++) { + int i = sel[j]; + long denom = vector2[i]; + hasDivBy0 = hasDivBy0 || (denom == 0); + if (denom != 0) { + outputVector[i] = vector1[i] % denom; + } + } + } else { + for(int i = 0; i != n; i++) { + hasDivBy0 = hasDivBy0 || (vector2[i] == 0); + if (vector2[i] != 0) { + outputVector[i] = vector1[i] % vector2[i]; + } + } + } + } + + /* For the case when the output can have null values, follow + * the convention that the data values must be 1 for long and + * NaN for double. This is to prevent possible later zero-divide errors + * in complex arithmetic expressions like col2 % (col1 - 1) + * in the case when some col1 entries are null. + */ + if (!hasDivBy0) { + NullUtil.setNullDataEntriesLong(outputColVector, batch.selectedInUse, sel, n); + } else { + NullUtil.setNullAndDivBy0DataEntriesLong( + outputColVector, batch.selectedInUse, sel, n, inputColVector2); + } + } + + @Override + public String vectorExpressionParameters() { + return getColumnParamString(0, colNum1) + ", " + getColumnParamString(1, colNum2); + } + + @Override + public VectorExpressionDescriptor.Descriptor getDescriptor() { + return (new VectorExpressionDescriptor.Builder()) + .setMode( + VectorExpressionDescriptor.Mode.PROJECTION) + .setNumArguments(2) + .setArgumentTypes( + VectorExpressionDescriptor.ArgumentType.getType("long"), + VectorExpressionDescriptor.ArgumentType.getType("long")) + .setInputExpressionTypes( + VectorExpressionDescriptor.InputExpressionType.COLUMN, + VectorExpressionDescriptor.InputExpressionType.COLUMN).build(); + } +} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPMod.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPMod.java index 6d3e82e9b96e012d875d947fa397c6c67df6a931..c85fdc84f4801f0d1ba8944f0e612e0875308add 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPMod.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPMod.java @@ -21,6 +21,7 @@ import org.apache.hadoop.hive.common.type.HiveDecimal; import org.apache.hadoop.hive.ql.exec.Description; import org.apache.hadoop.hive.ql.exec.vector.VectorizedExpressions; +import org.apache.hadoop.hive.ql.exec.vector.expressions.LongColModuloLongColumn; import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.*; import org.apache.hadoop.hive.serde2.io.ByteWritable; import org.apache.hadoop.hive.serde2.io.DoubleWritable; diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorizationContext.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorizationContext.java index 21f6540512ec795171014d87a6fde0d0ea5f23cf..9e1d64f6ac96cfc6a7656a6db8f51b147b18d6e7 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorizationContext.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorizationContext.java @@ -54,6 +54,7 @@ import org.apache.hadoop.hive.ql.exec.vector.expressions.IfExprVarCharScalarStringGroupColumn; import org.apache.hadoop.hive.ql.exec.vector.expressions.IsNotNull; import org.apache.hadoop.hive.ql.exec.vector.expressions.IsNull; +import org.apache.hadoop.hive.ql.exec.vector.expressions.LongColModuloLongColumn; import org.apache.hadoop.hive.ql.exec.vector.expressions.LongColumnInList; import org.apache.hadoop.hive.ql.exec.vector.expressions.LongColEqualLongScalar; import org.apache.hadoop.hive.ql.exec.vector.expressions.LongColGreaterLongScalar; @@ -107,7 +108,6 @@ import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FuncRoundDoubleToDouble; import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FuncSinDoubleToDouble; import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.LongColAddLongColumn; -import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.LongColModuloLongColumn; import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.LongColMultiplyLongColumn; import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.LongColSubtractLongColumn; import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.LongColUnaryMinus; diff --git a/ql/src/test/queries/clientpositive/vectorization_div0.q b/ql/src/test/queries/clientpositive/vectorization_div0.q index 025d457807dd0642965a81c6b093e421c4acd0f8..d7b6c3c4f403532b76d06700bca556ecb602747c 100644 --- a/ql/src/test/queries/clientpositive/vectorization_div0.q +++ b/ql/src/test/queries/clientpositive/vectorization_div0.q @@ -5,8 +5,8 @@ set hive.fetch.task.conversion=none; -- TODO: add more stuff here after HIVE-5918 is fixed, such as cbigint and constants explain vectorization expression -select cdouble / 0.0 from alltypesorc limit 100; -select cdouble / 0.0 from alltypesorc limit 100; +select cint / 0, ctinyint / 0, cbigint / 0, cdouble / 0.0 from alltypesorc limit 100; +select cint / 0, ctinyint / 0, cbigint / 0, cdouble / 0.0 from alltypesorc limit 100; -- There are no zeros in the table, but there is 988888, so use it as zero @@ -25,3 +25,11 @@ from alltypesorc where cdouble >= -500 and cdouble < -199 order by s1, s2 limit select (cdouble + 200.0) as s1, cbigint / (cdouble + 200.0) as s2, (cdouble + 200.0) / (cdouble + 200.0), cbigint / (cdouble + 200.0), 3 / (cdouble + 200.0), 1.2 / (cdouble + 200.0) from alltypesorc where cdouble >= -500 and cdouble < -199 order by s1, s2 limit 100; +-- There are no zeros in the table, but there is 1018195815 in cbigint, 528534767 in cint, so using it to do a divide by zero. ctinyint has a zero so can be used directly + +explain vectorization expression +select cint, cbigint, ctinyint, (cint / (cint - 528534767)) as c1, (cbigint / (cbigint - 1018195815)) as c2, (ctinyint / ctinyint) as c3, (cint % (cint - 528534767)) as c4, (cbigint % (cbigint - 1018195815)), (ctinyint % ctinyint) as c3 +from alltypesorc where cint > 500000000 or cdouble > 1000000000 or ctinyint = 0 order by c1, c2 limit 100; + +select cint, cbigint, ctinyint, (cint / (cint - 528534767)) as c1, (cbigint / (cbigint - 1018195815)) as c2, (ctinyint / ctinyint) as c3, (cint % (cint - 528534767)) as c4, (cbigint % (cbigint - 1018195815)), (ctinyint % ctinyint) as c3 +from alltypesorc where cint > 500000000 or cdouble > 1000000000 or ctinyint = 0 order by c1, c2 limit 100; diff --git a/ql/src/test/results/clientpositive/spark/vectorization_div0.q.out b/ql/src/test/results/clientpositive/spark/vectorization_div0.q.out index 631b0723fb0d7ab011ad2bfd7be4b33d11d76b1c..105f7c3b964fc620b13fa33c96c9c48328e82506 100644 --- a/ql/src/test/results/clientpositive/spark/vectorization_div0.q.out +++ b/ql/src/test/results/clientpositive/spark/vectorization_div0.q.out @@ -1,8 +1,8 @@ PREHOOK: query: explain vectorization expression -select cdouble / 0.0 from alltypesorc limit 100 +select cint / 0, ctinyint / 0, cbigint / 0, cdouble / 0.0 from alltypesorc limit 100 PREHOOK: type: QUERY POSTHOOK: query: explain vectorization expression -select cdouble / 0.0 from alltypesorc limit 100 +select cint / 0, ctinyint / 0, cbigint / 0, cdouble / 0.0 from alltypesorc limit 100 POSTHOOK: type: QUERY PLAN VECTORIZATION: enabled: true @@ -25,13 +25,13 @@ STAGE PLANS: TableScan Vectorization: native: true Select Operator - expressions: (cdouble / 0.0) (type: double) - outputColumnNames: _col0 + expressions: (UDFToDouble(cint) / 0.0) (type: double), (UDFToDouble(ctinyint) / 0.0) (type: double), (UDFToDouble(cbigint) / 0.0) (type: double), (cdouble / 0.0) (type: double) + outputColumnNames: _col0, _col1, _col2, _col3 Select Vectorization: className: VectorSelectOperator native: true - projectedOutputColumnNums: [13] - selectExpressions: DoubleColDivideDoubleScalar(col 5:double, val 0.0) -> 13:double + projectedOutputColumnNums: [14, 15, 16, 13] + selectExpressions: DoubleColDivideDoubleScalar(col 13:double, val 0.0)(children: CastLongToDouble(col 2:int) -> 13:double) -> 14:double, DoubleColDivideDoubleScalar(col 13:double, val 0.0)(children: CastLongToDouble(col 0:tinyint) -> 13:double) -> 15:double, DoubleColDivideDoubleScalar(col 13:double, val 0.0)(children: CastLongToDouble(col 3:bigint) -> 13:double) -> 16:double, DoubleColDivideDoubleScalar(col 5:double, val 0.0) -> 13:double Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE Limit Number of rows: 100 @@ -66,114 +66,114 @@ STAGE PLANS: Processor Tree: ListSink -PREHOOK: query: select cdouble / 0.0 from alltypesorc limit 100 +PREHOOK: query: select cint / 0, ctinyint / 0, cbigint / 0, cdouble / 0.0 from alltypesorc limit 100 PREHOOK: type: QUERY PREHOOK: Input: default@alltypesorc #### A masked pattern was here #### -POSTHOOK: query: select cdouble / 0.0 from alltypesorc limit 100 +POSTHOOK: query: select cint / 0, ctinyint / 0, cbigint / 0, cdouble / 0.0 from alltypesorc limit 100 POSTHOOK: type: QUERY POSTHOOK: Input: default@alltypesorc #### A masked pattern was here #### -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL PREHOOK: query: explain vectorization expression select (cbigint - 988888L) as s1, cdouble / (cbigint - 988888L) as s2, 1.2 / (cbigint - 988888L) from alltypesorc where cbigint > 0 and cbigint < 100000000 order by s1, s2 limit 100 @@ -606,3 +606,219 @@ POSTHOOK: Input: default@alltypesorc 0.0 NULL NULL NULL NULL NULL 0.0 NULL NULL NULL NULL NULL 0.0 NULL NULL NULL NULL NULL +PREHOOK: query: explain vectorization expression +select cint, cbigint, ctinyint, (cint / (cint - 528534767)) as c1, (cbigint / (cbigint - 1018195815)) as c2, (ctinyint / ctinyint) as c3, (cint % (cint - 528534767)) as c4, (cbigint % (cbigint - 1018195815)), (ctinyint % ctinyint) as c3 +from alltypesorc where cint > 500000000 or cdouble > 1000000000 or ctinyint = 0 order by c1, c2 limit 100 +PREHOOK: type: QUERY +POSTHOOK: query: explain vectorization expression +select cint, cbigint, ctinyint, (cint / (cint - 528534767)) as c1, (cbigint / (cbigint - 1018195815)) as c2, (ctinyint / ctinyint) as c3, (cint % (cint - 528534767)) as c4, (cbigint % (cbigint - 1018195815)), (ctinyint % ctinyint) as c3 +from alltypesorc where cint > 500000000 or cdouble > 1000000000 or ctinyint = 0 order by c1, c2 limit 100 +POSTHOOK: type: QUERY +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Spark + Edges: + Reducer 2 <- Map 1 (SORT, 1) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: alltypesorc + Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE + TableScan Vectorization: + native: true + Filter Operator + Filter Vectorization: + className: VectorFilterOperator + native: true + predicateExpression: FilterExprOrExpr(children: FilterLongColGreaterLongScalar(col 2:int, val 500000000), FilterDoubleColGreaterDoubleScalar(col 5:double, val 1.0E9), FilterLongColEqualLongScalar(col 0:tinyint, val 0)) + predicate: ((cdouble > 1.0E9) or (cint > 500000000) or (ctinyint = 0)) (type: boolean) + Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: cint (type: int), cbigint (type: bigint), ctinyint (type: tinyint), (cint / (cint - 528534767)) (type: double), (cbigint / (cbigint - 1018195815)) (type: double), (ctinyint / ctinyint) (type: double), (cint % (cint - 528534767)) (type: int), (cbigint % (cbigint - 1018195815)) (type: bigint), (ctinyint % ctinyint) (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 + Select Vectorization: + className: VectorSelectOperator + native: true + projectedOutputColumnNums: [2, 3, 0, 14, 15, 16, 17, 18, 13] + selectExpressions: LongColDivideLongColumn(col 2:int, col 13:int)(children: LongColSubtractLongScalar(col 2:int, val 528534767) -> 13:int) -> 14:double, LongColDivideLongColumn(col 3:bigint, col 13:bigint)(children: LongColSubtractLongScalar(col 3:bigint, val 1018195815) -> 13:bigint) -> 15:double, LongColDivideLongColumn(col 0:tinyint, col 0:tinyint) -> 16:double, LongColModuloLongColumn(col 2:int, col 13:int)(children: LongColSubtractLongScalar(col 2:int, val 528534767) -> 13:int) -> 17:int, LongColModuloLongColumn(col 3:bigint, col 13:bigint)(children: LongColSubtractLongScalar(col 3:bigint, val 1018195815) -> 13:bigint) -> 18:bigint, LongColModuloLongColumn(col 0:tinyint, col 0:tinyint) -> 13:tinyint + Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col3 (type: double), _col4 (type: double) + sort order: ++ + Reduce Sink Vectorization: + className: VectorReduceSinkObjectHashOperator + native: true + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine spark IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE + TopN Hash Memory Usage: 0.1 + value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: tinyint), _col5 (type: double), _col6 (type: int), _col7 (type: bigint), _col8 (type: tinyint) + Execution mode: vectorized + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true + inputFormatFeatureSupport: [] + featureSupportInUse: [] + inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + allNative: true + usesVectorUDFAdaptor: false + vectorized: true + Reducer 2 + Execution mode: vectorized + Reduce Vectorization: + enabled: true + enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine spark IN [tez, spark] IS true + allNative: false + usesVectorUDFAdaptor: false + vectorized: true + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: int), VALUE._col1 (type: bigint), VALUE._col2 (type: tinyint), KEY.reducesinkkey0 (type: double), KEY.reducesinkkey1 (type: double), VALUE._col3 (type: double), VALUE._col4 (type: int), VALUE._col5 (type: bigint), VALUE._col6 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 + Select Vectorization: + className: VectorSelectOperator + native: true + projectedOutputColumnNums: [2, 3, 4, 0, 1, 5, 6, 7, 8] + Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 100 + Limit Vectorization: + className: VectorLimitOperator + native: true + Statistics: Num rows: 100 Data size: 21500 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + File Sink Vectorization: + className: VectorFileSinkOperator + native: false + Statistics: Num rows: 100 Data size: 21500 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: 100 + Processor Tree: + ListSink + +PREHOOK: query: select cint, cbigint, ctinyint, (cint / (cint - 528534767)) as c1, (cbigint / (cbigint - 1018195815)) as c2, (ctinyint / ctinyint) as c3, (cint % (cint - 528534767)) as c4, (cbigint % (cbigint - 1018195815)), (ctinyint % ctinyint) as c3 +from alltypesorc where cint > 500000000 or cdouble > 1000000000 or ctinyint = 0 order by c1, c2 limit 100 +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypesorc +#### A masked pattern was here #### +POSTHOOK: query: select cint, cbigint, ctinyint, (cint / (cint - 528534767)) as c1, (cbigint / (cbigint - 1018195815)) as c2, (ctinyint / ctinyint) as c3, (cint % (cint - 528534767)) as c4, (cbigint % (cbigint - 1018195815)), (ctinyint % ctinyint) as c3 +from alltypesorc where cint > 500000000 or cdouble > 1000000000 or ctinyint = 0 order by c1, c2 limit 100 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alltypesorc +#### A masked pattern was here #### +528534767 NULL -50 NULL NULL 1.0 NULL NULL 0 +528534767 NULL 33 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -28 NULL NULL 1.0 NULL NULL 0 +528534767 NULL 31 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -34 NULL NULL 1.0 NULL NULL 0 +528534767 NULL 29 NULL NULL 1.0 NULL NULL 0 +528534767 NULL 31 NULL NULL 1.0 NULL NULL 0 +528534767 NULL NULL NULL NULL NULL NULL NULL NULL +528534767 NULL -11 NULL NULL 1.0 NULL NULL 0 +528534767 NULL 61 NULL NULL 1.0 NULL NULL 0 +528534767 NULL 16 NULL NULL 1.0 NULL NULL 0 +528534767 NULL 62 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -23 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -51 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -11 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -48 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -62 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -45 NULL NULL 1.0 NULL NULL 0 +528534767 NULL 40 NULL NULL 1.0 NULL NULL 0 +528534767 NULL 39 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -32 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -56 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -7 NULL NULL 1.0 NULL NULL 0 +528534767 NULL 24 NULL NULL 1.0 NULL NULL 0 +528534767 NULL 36 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -23 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -55 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -11 NULL NULL 1.0 NULL NULL 0 +528534767 NULL 51 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -24 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -44 NULL NULL 1.0 NULL NULL 0 +NULL 1018195815 0 NULL NULL NULL NULL NULL NULL +528534767 NULL 24 NULL NULL 1.0 NULL NULL 0 +528534767 NULL 4 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -57 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -22 NULL NULL 1.0 NULL NULL 0 +528534767 NULL 28 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -16 NULL NULL 1.0 NULL NULL 0 +528534767 NULL 46 NULL NULL 1.0 NULL NULL 0 +528534767 NULL 29 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -56 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -16 NULL NULL 1.0 NULL NULL 0 +528534767 NULL 38 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -54 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -23 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -19 NULL NULL 1.0 NULL NULL 0 +528534767 NULL 40 NULL NULL 1.0 NULL NULL 0 +528534767 NULL 53 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -34 NULL NULL 1.0 NULL NULL 0 +528534767 NULL 5 NULL NULL 1.0 NULL NULL 0 +528534767 NULL 51 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -4 NULL NULL 1.0 NULL NULL 0 +528534767 NULL 61 NULL NULL 1.0 NULL NULL 0 +528534767 NULL 19 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -33 NULL NULL 1.0 NULL NULL 0 +528534767 NULL 53 NULL NULL 1.0 NULL NULL 0 +528534767 NULL 18 NULL NULL 1.0 NULL NULL 0 +528534767 NULL 30 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -36 NULL NULL 1.0 NULL NULL 0 +528534767 NULL 34 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -55 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -40 NULL NULL 1.0 NULL NULL 0 +528534767 NULL 21 NULL NULL 1.0 NULL NULL 0 +528534767 NULL 61 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -59 NULL NULL 1.0 NULL NULL 0 +528534767 NULL 0 NULL NULL NULL NULL NULL NULL +528534767 NULL -21 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -33 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -30 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -5 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -53 NULL NULL 1.0 NULL NULL 0 +528534767 NULL 34 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -5 NULL NULL 1.0 NULL NULL 0 +528534767 NULL 27 NULL NULL 1.0 NULL NULL 0 +528534767 NULL NULL NULL NULL NULL NULL NULL NULL +528534767 NULL -21 NULL NULL 1.0 NULL NULL 0 +528534767 NULL 43 NULL NULL 1.0 NULL NULL 0 +528534767 NULL 41 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -28 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -5 NULL NULL 1.0 NULL NULL 0 +528534767 NULL 13 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -45 NULL NULL 1.0 NULL NULL 0 +528534767 NULL 10 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -22 NULL NULL 1.0 NULL NULL 0 +528534767 NULL 38 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -48 NULL NULL 1.0 NULL NULL 0 +528534767 NULL 2 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -37 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -43 NULL NULL 1.0 NULL NULL 0 +528534767 NULL 36 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -1 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -12 NULL NULL 1.0 NULL NULL 0 +528534767 NULL 0 NULL NULL NULL NULL NULL NULL +528534767 NULL 26 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -22 NULL NULL 1.0 NULL NULL 0 +528534767 NULL 9 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -13 NULL NULL 1.0 NULL NULL 0 +528534767 NULL 38 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -4 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -47 NULL NULL 1.0 NULL NULL 0 diff --git a/ql/src/test/results/clientpositive/tez/vectorization_div0.q.out b/ql/src/test/results/clientpositive/tez/vectorization_div0.q.out index 6c3354cb4a8cd439d86df7e6b0cf759ea4c04cd0..81e901970939fd5c0f93215123a5703e51ba17f2 100644 --- a/ql/src/test/results/clientpositive/tez/vectorization_div0.q.out +++ b/ql/src/test/results/clientpositive/tez/vectorization_div0.q.out @@ -1,8 +1,8 @@ PREHOOK: query: explain vectorization expression -select cdouble / 0.0 from alltypesorc limit 100 +select cint / 0, ctinyint / 0, cbigint / 0, cdouble / 0.0 from alltypesorc limit 100 PREHOOK: type: QUERY POSTHOOK: query: explain vectorization expression -select cdouble / 0.0 from alltypesorc limit 100 +select cint / 0, ctinyint / 0, cbigint / 0, cdouble / 0.0 from alltypesorc limit 100 POSTHOOK: type: QUERY PLAN VECTORIZATION: enabled: true @@ -21,30 +21,30 @@ STAGE PLANS: Map Operator Tree: TableScan alias: alltypesorc - Statistics: Num rows: 12288 Data size: 73400 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 12288 Data size: 220184 Basic stats: COMPLETE Column stats: COMPLETE TableScan Vectorization: native: true Select Operator - expressions: (cdouble / 0.0) (type: double) - outputColumnNames: _col0 + expressions: (UDFToDouble(cint) / 0.0) (type: double), (UDFToDouble(ctinyint) / 0.0) (type: double), (UDFToDouble(cbigint) / 0.0) (type: double), (cdouble / 0.0) (type: double) + outputColumnNames: _col0, _col1, _col2, _col3 Select Vectorization: className: VectorSelectOperator native: true - projectedOutputColumnNums: [13] - selectExpressions: DoubleColDivideDoubleScalar(col 5:double, val 0.0) -> 13:double - Statistics: Num rows: 12288 Data size: 98304 Basic stats: COMPLETE Column stats: COMPLETE + projectedOutputColumnNums: [14, 15, 16, 13] + selectExpressions: DoubleColDivideDoubleScalar(col 13:double, val 0.0)(children: CastLongToDouble(col 2:int) -> 13:double) -> 14:double, DoubleColDivideDoubleScalar(col 13:double, val 0.0)(children: CastLongToDouble(col 0:tinyint) -> 13:double) -> 15:double, DoubleColDivideDoubleScalar(col 13:double, val 0.0)(children: CastLongToDouble(col 3:bigint) -> 13:double) -> 16:double, DoubleColDivideDoubleScalar(col 5:double, val 0.0) -> 13:double + Statistics: Num rows: 12288 Data size: 393216 Basic stats: COMPLETE Column stats: COMPLETE Limit Number of rows: 100 Limit Vectorization: className: VectorLimitOperator native: true - Statistics: Num rows: 100 Data size: 800 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 100 Data size: 3200 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false File Sink Vectorization: className: VectorFileSinkOperator native: false - Statistics: Num rows: 100 Data size: 800 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 100 Data size: 3200 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -66,114 +66,114 @@ STAGE PLANS: Processor Tree: ListSink -PREHOOK: query: select cdouble / 0.0 from alltypesorc limit 100 +PREHOOK: query: select cint / 0, ctinyint / 0, cbigint / 0, cdouble / 0.0 from alltypesorc limit 100 PREHOOK: type: QUERY PREHOOK: Input: default@alltypesorc #### A masked pattern was here #### -POSTHOOK: query: select cdouble / 0.0 from alltypesorc limit 100 +POSTHOOK: query: select cint / 0, ctinyint / 0, cbigint / 0, cdouble / 0.0 from alltypesorc limit 100 POSTHOOK: type: QUERY POSTHOOK: Input: default@alltypesorc #### A masked pattern was here #### -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL PREHOOK: query: explain vectorization expression select (cbigint - 988888L) as s1, cdouble / (cbigint - 988888L) as s2, 1.2 / (cbigint - 988888L) from alltypesorc where cbigint > 0 and cbigint < 100000000 order by s1, s2 limit 100 @@ -608,3 +608,220 @@ POSTHOOK: Input: default@alltypesorc 0.0 NULL NULL NULL NULL NULL 0.0 NULL NULL NULL NULL NULL 0.0 NULL NULL NULL NULL NULL +PREHOOK: query: explain vectorization expression +select cint, cbigint, ctinyint, (cint / (cint - 528534767)) as c1, (cbigint / (cbigint - 1018195815)) as c2, (ctinyint / ctinyint) as c3, (cint % (cint - 528534767)) as c4, (cbigint % (cbigint - 1018195815)), (ctinyint % ctinyint) as c3 +from alltypesorc where cint > 500000000 or cdouble > 1000000000 or ctinyint = 0 order by c1, c2 limit 100 +PREHOOK: type: QUERY +POSTHOOK: query: explain vectorization expression +select cint, cbigint, ctinyint, (cint / (cint - 528534767)) as c1, (cbigint / (cbigint - 1018195815)) as c2, (ctinyint / ctinyint) as c3, (cint % (cint - 528534767)) as c4, (cbigint % (cbigint - 1018195815)), (ctinyint % ctinyint) as c3 +from alltypesorc where cint > 500000000 or cdouble > 1000000000 or ctinyint = 0 order by c1, c2 limit 100 +POSTHOOK: type: QUERY +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: alltypesorc + Statistics: Num rows: 12288 Data size: 220184 Basic stats: COMPLETE Column stats: COMPLETE + TableScan Vectorization: + native: true + Filter Operator + Filter Vectorization: + className: VectorFilterOperator + native: true + predicateExpression: FilterExprOrExpr(children: FilterLongColGreaterLongScalar(col 2:int, val 500000000), FilterDoubleColGreaterDoubleScalar(col 5:double, val 1.0E9), FilterLongColEqualLongScalar(col 0:tinyint, val 0)) + predicate: ((cdouble > 1.0E9) or (cint > 500000000) or (ctinyint = 0)) (type: boolean) + Statistics: Num rows: 4193 Data size: 75144 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: cint (type: int), cbigint (type: bigint), ctinyint (type: tinyint), (cint / (cint - 528534767)) (type: double), (cbigint / (cbigint - 1018195815)) (type: double), (ctinyint / ctinyint) (type: double), (cint % (cint - 528534767)) (type: int), (cbigint % (cbigint - 1018195815)) (type: bigint), (ctinyint % ctinyint) (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 + Select Vectorization: + className: VectorSelectOperator + native: true + projectedOutputColumnNums: [2, 3, 0, 14, 15, 16, 17, 18, 13] + selectExpressions: LongColDivideLongColumn(col 2:int, col 13:int)(children: LongColSubtractLongScalar(col 2:int, val 528534767) -> 13:int) -> 14:double, LongColDivideLongColumn(col 3:bigint, col 13:bigint)(children: LongColSubtractLongScalar(col 3:bigint, val 1018195815) -> 13:bigint) -> 15:double, LongColDivideLongColumn(col 0:tinyint, col 0:tinyint) -> 16:double, LongColModuloLongColumn(col 2:int, col 13:int)(children: LongColSubtractLongScalar(col 2:int, val 528534767) -> 13:int) -> 17:int, LongColModuloLongColumn(col 3:bigint, col 13:bigint)(children: LongColSubtractLongScalar(col 3:bigint, val 1018195815) -> 13:bigint) -> 18:bigint, LongColModuloLongColumn(col 0:tinyint, col 0:tinyint) -> 13:tinyint + Statistics: Num rows: 4193 Data size: 217816 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: _col3 (type: double), _col4 (type: double) + sort order: ++ + Reduce Sink Vectorization: + className: VectorReduceSinkObjectHashOperator + native: true + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + Statistics: Num rows: 4193 Data size: 217816 Basic stats: COMPLETE Column stats: COMPLETE + TopN Hash Memory Usage: 0.1 + value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: tinyint), _col5 (type: double), _col6 (type: int), _col7 (type: bigint), _col8 (type: tinyint) + Execution mode: vectorized + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true + inputFormatFeatureSupport: [] + featureSupportInUse: [] + inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + allNative: true + usesVectorUDFAdaptor: false + vectorized: true + Reducer 2 + Execution mode: vectorized + Reduce Vectorization: + enabled: true + enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true + allNative: false + usesVectorUDFAdaptor: false + vectorized: true + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: int), VALUE._col1 (type: bigint), VALUE._col2 (type: tinyint), KEY.reducesinkkey0 (type: double), KEY.reducesinkkey1 (type: double), VALUE._col3 (type: double), VALUE._col4 (type: int), VALUE._col5 (type: bigint), VALUE._col6 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 + Select Vectorization: + className: VectorSelectOperator + native: true + projectedOutputColumnNums: [2, 3, 4, 0, 1, 5, 6, 7, 8] + Statistics: Num rows: 4193 Data size: 217816 Basic stats: COMPLETE Column stats: COMPLETE + Limit + Number of rows: 100 + Limit Vectorization: + className: VectorLimitOperator + native: true + Statistics: Num rows: 100 Data size: 5216 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + File Sink Vectorization: + className: VectorFileSinkOperator + native: false + Statistics: Num rows: 100 Data size: 5216 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: 100 + Processor Tree: + ListSink + +PREHOOK: query: select cint, cbigint, ctinyint, (cint / (cint - 528534767)) as c1, (cbigint / (cbigint - 1018195815)) as c2, (ctinyint / ctinyint) as c3, (cint % (cint - 528534767)) as c4, (cbigint % (cbigint - 1018195815)), (ctinyint % ctinyint) as c3 +from alltypesorc where cint > 500000000 or cdouble > 1000000000 or ctinyint = 0 order by c1, c2 limit 100 +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypesorc +#### A masked pattern was here #### +POSTHOOK: query: select cint, cbigint, ctinyint, (cint / (cint - 528534767)) as c1, (cbigint / (cbigint - 1018195815)) as c2, (ctinyint / ctinyint) as c3, (cint % (cint - 528534767)) as c4, (cbigint % (cbigint - 1018195815)), (ctinyint % ctinyint) as c3 +from alltypesorc where cint > 500000000 or cdouble > 1000000000 or ctinyint = 0 order by c1, c2 limit 100 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alltypesorc +#### A masked pattern was here #### +528534767 NULL -50 NULL NULL 1.0 NULL NULL 0 +528534767 NULL 33 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -28 NULL NULL 1.0 NULL NULL 0 +528534767 NULL 31 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -34 NULL NULL 1.0 NULL NULL 0 +528534767 NULL 29 NULL NULL 1.0 NULL NULL 0 +528534767 NULL 31 NULL NULL 1.0 NULL NULL 0 +528534767 NULL NULL NULL NULL NULL NULL NULL NULL +528534767 NULL -11 NULL NULL 1.0 NULL NULL 0 +528534767 NULL 61 NULL NULL 1.0 NULL NULL 0 +528534767 NULL 16 NULL NULL 1.0 NULL NULL 0 +528534767 NULL 62 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -23 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -51 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -11 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -48 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -62 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -45 NULL NULL 1.0 NULL NULL 0 +528534767 NULL 40 NULL NULL 1.0 NULL NULL 0 +528534767 NULL 39 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -32 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -56 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -7 NULL NULL 1.0 NULL NULL 0 +528534767 NULL 24 NULL NULL 1.0 NULL NULL 0 +528534767 NULL 36 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -23 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -55 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -11 NULL NULL 1.0 NULL NULL 0 +528534767 NULL 51 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -24 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -44 NULL NULL 1.0 NULL NULL 0 +NULL 1018195815 0 NULL NULL NULL NULL NULL NULL +528534767 NULL 24 NULL NULL 1.0 NULL NULL 0 +528534767 NULL 4 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -57 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -22 NULL NULL 1.0 NULL NULL 0 +528534767 NULL 28 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -16 NULL NULL 1.0 NULL NULL 0 +528534767 NULL 46 NULL NULL 1.0 NULL NULL 0 +528534767 NULL 29 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -56 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -16 NULL NULL 1.0 NULL NULL 0 +528534767 NULL 38 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -54 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -23 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -19 NULL NULL 1.0 NULL NULL 0 +528534767 NULL 40 NULL NULL 1.0 NULL NULL 0 +528534767 NULL 53 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -34 NULL NULL 1.0 NULL NULL 0 +528534767 NULL 5 NULL NULL 1.0 NULL NULL 0 +528534767 NULL 51 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -4 NULL NULL 1.0 NULL NULL 0 +528534767 NULL 61 NULL NULL 1.0 NULL NULL 0 +528534767 NULL 19 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -33 NULL NULL 1.0 NULL NULL 0 +528534767 NULL 53 NULL NULL 1.0 NULL NULL 0 +528534767 NULL 18 NULL NULL 1.0 NULL NULL 0 +528534767 NULL 30 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -36 NULL NULL 1.0 NULL NULL 0 +528534767 NULL 34 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -55 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -40 NULL NULL 1.0 NULL NULL 0 +528534767 NULL 21 NULL NULL 1.0 NULL NULL 0 +528534767 NULL 61 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -59 NULL NULL 1.0 NULL NULL 0 +528534767 NULL 0 NULL NULL NULL NULL NULL NULL +528534767 NULL -21 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -33 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -30 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -5 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -53 NULL NULL 1.0 NULL NULL 0 +528534767 NULL 34 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -5 NULL NULL 1.0 NULL NULL 0 +528534767 NULL 27 NULL NULL 1.0 NULL NULL 0 +528534767 NULL NULL NULL NULL NULL NULL NULL NULL +528534767 NULL -21 NULL NULL 1.0 NULL NULL 0 +528534767 NULL 43 NULL NULL 1.0 NULL NULL 0 +528534767 NULL 41 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -28 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -5 NULL NULL 1.0 NULL NULL 0 +528534767 NULL 13 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -45 NULL NULL 1.0 NULL NULL 0 +528534767 NULL 10 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -22 NULL NULL 1.0 NULL NULL 0 +528534767 NULL 38 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -48 NULL NULL 1.0 NULL NULL 0 +528534767 NULL 2 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -37 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -43 NULL NULL 1.0 NULL NULL 0 +528534767 NULL 36 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -1 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -12 NULL NULL 1.0 NULL NULL 0 +528534767 NULL 0 NULL NULL NULL NULL NULL NULL +528534767 NULL 26 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -22 NULL NULL 1.0 NULL NULL 0 +528534767 NULL 9 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -13 NULL NULL 1.0 NULL NULL 0 +528534767 NULL 38 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -4 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -47 NULL NULL 1.0 NULL NULL 0 diff --git a/ql/src/test/results/clientpositive/vectorization_div0.q.out b/ql/src/test/results/clientpositive/vectorization_div0.q.out index 97f1687b85193e681f26c61107a6d9266c1d87a2..64c05c70652f26c79f9a5af9937f873ff97a9ba3 100644 --- a/ql/src/test/results/clientpositive/vectorization_div0.q.out +++ b/ql/src/test/results/clientpositive/vectorization_div0.q.out @@ -1,8 +1,8 @@ PREHOOK: query: explain vectorization expression -select cdouble / 0.0 from alltypesorc limit 100 +select cint / 0, ctinyint / 0, cbigint / 0, cdouble / 0.0 from alltypesorc limit 100 PREHOOK: type: QUERY POSTHOOK: query: explain vectorization expression -select cdouble / 0.0 from alltypesorc limit 100 +select cint / 0, ctinyint / 0, cbigint / 0, cdouble / 0.0 from alltypesorc limit 100 POSTHOOK: type: QUERY PLAN VECTORIZATION: enabled: true @@ -22,13 +22,13 @@ STAGE PLANS: TableScan Vectorization: native: true Select Operator - expressions: (cdouble / 0.0) (type: double) - outputColumnNames: _col0 + expressions: (UDFToDouble(cint) / 0.0) (type: double), (UDFToDouble(ctinyint) / 0.0) (type: double), (UDFToDouble(cbigint) / 0.0) (type: double), (cdouble / 0.0) (type: double) + outputColumnNames: _col0, _col1, _col2, _col3 Select Vectorization: className: VectorSelectOperator native: true - projectedOutputColumnNums: [13] - selectExpressions: DoubleColDivideDoubleScalar(col 5:double, val 0.0) -> 13:double + projectedOutputColumnNums: [14, 15, 16, 13] + selectExpressions: DoubleColDivideDoubleScalar(col 13:double, val 0.0)(children: CastLongToDouble(col 2:int) -> 13:double) -> 14:double, DoubleColDivideDoubleScalar(col 13:double, val 0.0)(children: CastLongToDouble(col 0:tinyint) -> 13:double) -> 15:double, DoubleColDivideDoubleScalar(col 13:double, val 0.0)(children: CastLongToDouble(col 3:bigint) -> 13:double) -> 16:double, DoubleColDivideDoubleScalar(col 5:double, val 0.0) -> 13:double Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE Limit Number of rows: 100 @@ -63,114 +63,114 @@ STAGE PLANS: Processor Tree: ListSink -PREHOOK: query: select cdouble / 0.0 from alltypesorc limit 100 +PREHOOK: query: select cint / 0, ctinyint / 0, cbigint / 0, cdouble / 0.0 from alltypesorc limit 100 PREHOOK: type: QUERY PREHOOK: Input: default@alltypesorc #### A masked pattern was here #### -POSTHOOK: query: select cdouble / 0.0 from alltypesorc limit 100 +POSTHOOK: query: select cint / 0, ctinyint / 0, cbigint / 0, cdouble / 0.0 from alltypesorc limit 100 POSTHOOK: type: QUERY POSTHOOK: Input: default@alltypesorc #### A masked pattern was here #### -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL -NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL PREHOOK: query: explain vectorization expression select (cbigint - 988888L) as s1, cdouble / (cbigint - 988888L) as s2, 1.2 / (cbigint - 988888L) from alltypesorc where cbigint > 0 and cbigint < 100000000 order by s1, s2 limit 100 @@ -567,3 +567,201 @@ POSTHOOK: Input: default@alltypesorc 0.0 NULL NULL NULL NULL NULL 0.0 NULL NULL NULL NULL NULL 0.0 NULL NULL NULL NULL NULL +PREHOOK: query: explain vectorization expression +select cint, cbigint, ctinyint, (cint / (cint - 528534767)) as c1, (cbigint / (cbigint - 1018195815)) as c2, (ctinyint / ctinyint) as c3, (cint % (cint - 528534767)) as c4, (cbigint % (cbigint - 1018195815)), (ctinyint % ctinyint) as c3 +from alltypesorc where cint > 500000000 or cdouble > 1000000000 or ctinyint = 0 order by c1, c2 limit 100 +PREHOOK: type: QUERY +POSTHOOK: query: explain vectorization expression +select cint, cbigint, ctinyint, (cint / (cint - 528534767)) as c1, (cbigint / (cbigint - 1018195815)) as c2, (ctinyint / ctinyint) as c3, (cint % (cint - 528534767)) as c4, (cbigint % (cbigint - 1018195815)), (ctinyint % ctinyint) as c3 +from alltypesorc where cint > 500000000 or cdouble > 1000000000 or ctinyint = 0 order by c1, c2 limit 100 +POSTHOOK: type: QUERY +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: alltypesorc + Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE + TableScan Vectorization: + native: true + Filter Operator + Filter Vectorization: + className: VectorFilterOperator + native: true + predicateExpression: FilterExprOrExpr(children: FilterLongColGreaterLongScalar(col 2:int, val 500000000), FilterDoubleColGreaterDoubleScalar(col 5:double, val 1.0E9), FilterLongColEqualLongScalar(col 0:tinyint, val 0)) + predicate: ((cdouble > 1.0E9) or (cint > 500000000) or (ctinyint = 0)) (type: boolean) + Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: cint (type: int), cbigint (type: bigint), ctinyint (type: tinyint), (cint / (cint - 528534767)) (type: double), (cbigint / (cbigint - 1018195815)) (type: double), (ctinyint / ctinyint) (type: double), (cint % (cint - 528534767)) (type: int), (cbigint % (cbigint - 1018195815)) (type: bigint), (ctinyint % ctinyint) (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 + Select Vectorization: + className: VectorSelectOperator + native: true + projectedOutputColumnNums: [2, 3, 0, 14, 15, 16, 17, 18, 13] + selectExpressions: LongColDivideLongColumn(col 2:int, col 13:int)(children: LongColSubtractLongScalar(col 2:int, val 528534767) -> 13:int) -> 14:double, LongColDivideLongColumn(col 3:bigint, col 13:bigint)(children: LongColSubtractLongScalar(col 3:bigint, val 1018195815) -> 13:bigint) -> 15:double, LongColDivideLongColumn(col 0:tinyint, col 0:tinyint) -> 16:double, LongColModuloLongColumn(col 2:int, col 13:int)(children: LongColSubtractLongScalar(col 2:int, val 528534767) -> 13:int) -> 17:int, LongColModuloLongColumn(col 3:bigint, col 13:bigint)(children: LongColSubtractLongScalar(col 3:bigint, val 1018195815) -> 13:bigint) -> 18:bigint, LongColModuloLongColumn(col 0:tinyint, col 0:tinyint) -> 13:tinyint + Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col3 (type: double), _col4 (type: double) + sort order: ++ + Reduce Sink Vectorization: + className: VectorReduceSinkOperator + native: false + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + nativeConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false + Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE + TopN Hash Memory Usage: 0.1 + value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: tinyint), _col5 (type: double), _col6 (type: int), _col7 (type: bigint), _col8 (type: tinyint) + Execution mode: vectorized + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true + inputFormatFeatureSupport: [] + featureSupportInUse: [] + inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + allNative: false + usesVectorUDFAdaptor: false + vectorized: true + Reduce Vectorization: + enabled: false + enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true + enableConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: int), VALUE._col1 (type: bigint), VALUE._col2 (type: tinyint), KEY.reducesinkkey0 (type: double), KEY.reducesinkkey1 (type: double), VALUE._col3 (type: double), VALUE._col4 (type: int), VALUE._col5 (type: bigint), VALUE._col6 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 + Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 100 + Statistics: Num rows: 100 Data size: 21500 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 100 Data size: 21500 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: 100 + Processor Tree: + ListSink + +PREHOOK: query: select cint, cbigint, ctinyint, (cint / (cint - 528534767)) as c1, (cbigint / (cbigint - 1018195815)) as c2, (ctinyint / ctinyint) as c3, (cint % (cint - 528534767)) as c4, (cbigint % (cbigint - 1018195815)), (ctinyint % ctinyint) as c3 +from alltypesorc where cint > 500000000 or cdouble > 1000000000 or ctinyint = 0 order by c1, c2 limit 100 +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypesorc +#### A masked pattern was here #### +POSTHOOK: query: select cint, cbigint, ctinyint, (cint / (cint - 528534767)) as c1, (cbigint / (cbigint - 1018195815)) as c2, (ctinyint / ctinyint) as c3, (cint % (cint - 528534767)) as c4, (cbigint % (cbigint - 1018195815)), (ctinyint % ctinyint) as c3 +from alltypesorc where cint > 500000000 or cdouble > 1000000000 or ctinyint = 0 order by c1, c2 limit 100 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alltypesorc +#### A masked pattern was here #### +528534767 NULL -47 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -4 NULL NULL 1.0 NULL NULL 0 +528534767 NULL 38 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -13 NULL NULL 1.0 NULL NULL 0 +528534767 NULL 9 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -22 NULL NULL 1.0 NULL NULL 0 +528534767 NULL 26 NULL NULL 1.0 NULL NULL 0 +528534767 NULL 0 NULL NULL NULL NULL NULL NULL +528534767 NULL -12 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -1 NULL NULL 1.0 NULL NULL 0 +528534767 NULL 36 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -43 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -37 NULL NULL 1.0 NULL NULL 0 +528534767 NULL 2 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -48 NULL NULL 1.0 NULL NULL 0 +528534767 NULL 38 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -22 NULL NULL 1.0 NULL NULL 0 +528534767 NULL 10 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -45 NULL NULL 1.0 NULL NULL 0 +528534767 NULL 13 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -5 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -28 NULL NULL 1.0 NULL NULL 0 +528534767 NULL 41 NULL NULL 1.0 NULL NULL 0 +528534767 NULL 43 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -21 NULL NULL 1.0 NULL NULL 0 +528534767 NULL NULL NULL NULL NULL NULL NULL NULL +528534767 NULL 27 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -5 NULL NULL 1.0 NULL NULL 0 +528534767 NULL 34 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -53 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -5 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -30 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -33 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -21 NULL NULL 1.0 NULL NULL 0 +528534767 NULL 0 NULL NULL NULL NULL NULL NULL +528534767 NULL -59 NULL NULL 1.0 NULL NULL 0 +528534767 NULL 61 NULL NULL 1.0 NULL NULL 0 +528534767 NULL 21 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -40 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -55 NULL NULL 1.0 NULL NULL 0 +528534767 NULL 34 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -36 NULL NULL 1.0 NULL NULL 0 +528534767 NULL 30 NULL NULL 1.0 NULL NULL 0 +528534767 NULL 18 NULL NULL 1.0 NULL NULL 0 +528534767 NULL 53 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -33 NULL NULL 1.0 NULL NULL 0 +528534767 NULL 19 NULL NULL 1.0 NULL NULL 0 +528534767 NULL 61 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -4 NULL NULL 1.0 NULL NULL 0 +528534767 NULL 51 NULL NULL 1.0 NULL NULL 0 +528534767 NULL 5 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -34 NULL NULL 1.0 NULL NULL 0 +528534767 NULL 53 NULL NULL 1.0 NULL NULL 0 +528534767 NULL 40 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -19 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -23 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -54 NULL NULL 1.0 NULL NULL 0 +528534767 NULL 38 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -16 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -56 NULL NULL 1.0 NULL NULL 0 +528534767 NULL 29 NULL NULL 1.0 NULL NULL 0 +528534767 NULL 46 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -16 NULL NULL 1.0 NULL NULL 0 +528534767 NULL 28 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -22 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -57 NULL NULL 1.0 NULL NULL 0 +528534767 NULL 4 NULL NULL 1.0 NULL NULL 0 +528534767 NULL 24 NULL NULL 1.0 NULL NULL 0 +NULL 1018195815 0 NULL NULL NULL NULL NULL NULL +528534767 NULL -44 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -24 NULL NULL 1.0 NULL NULL 0 +528534767 NULL 51 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -11 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -55 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -23 NULL NULL 1.0 NULL NULL 0 +528534767 NULL 36 NULL NULL 1.0 NULL NULL 0 +528534767 NULL 24 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -7 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -56 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -32 NULL NULL 1.0 NULL NULL 0 +528534767 NULL 39 NULL NULL 1.0 NULL NULL 0 +528534767 NULL 40 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -45 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -62 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -48 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -11 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -51 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -23 NULL NULL 1.0 NULL NULL 0 +528534767 NULL 62 NULL NULL 1.0 NULL NULL 0 +528534767 NULL 16 NULL NULL 1.0 NULL NULL 0 +528534767 NULL 61 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -11 NULL NULL 1.0 NULL NULL 0 +528534767 NULL NULL NULL NULL NULL NULL NULL NULL +528534767 NULL 31 NULL NULL 1.0 NULL NULL 0 +528534767 NULL 29 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -34 NULL NULL 1.0 NULL NULL 0 +528534767 NULL 31 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -28 NULL NULL 1.0 NULL NULL 0 +528534767 NULL 33 NULL NULL 1.0 NULL NULL 0 +528534767 NULL -50 NULL NULL 1.0 NULL NULL 0 diff --git a/vector-code-gen/src/org/apache/hadoop/hive/tools/GenVectorCode.java b/vector-code-gen/src/org/apache/hadoop/hive/tools/GenVectorCode.java index e58d4e91938dc266111042fe98b05a3d9c6fc5e9..c140f1275ba58d45f17bd05eb8d32af50a1b4f13 100644 --- a/vector-code-gen/src/org/apache/hadoop/hive/tools/GenVectorCode.java +++ b/vector-code-gen/src/org/apache/hadoop/hive/tools/GenVectorCode.java @@ -232,7 +232,6 @@ {"ScalarDivideColumn", "Modulo", "long", "double", "%"}, {"ScalarDivideColumn", "Modulo", "double", "long", "%"}, {"ScalarDivideColumn", "Modulo", "double", "double", "%"}, - {"ColumnDivideColumn", "Modulo", "long", "long", "%"}, {"ColumnDivideColumn", "Modulo", "long", "double", "%"}, {"ColumnDivideColumn", "Modulo", "double", "long", "%"}, {"ColumnDivideColumn", "Modulo", "double", "double", "%"},