diff --git ql/src/gen/vectorization/ExpressionTemplates/ColumnDivideScalarDecimal.txt ql/src/gen/vectorization/ExpressionTemplates/ColumnDivideScalarDecimal.txt index 0bd7c00421..1a1918dd92 100644 --- ql/src/gen/vectorization/ExpressionTemplates/ColumnDivideScalarDecimal.txt +++ ql/src/gen/vectorization/ExpressionTemplates/ColumnDivideScalarDecimal.txt @@ -72,7 +72,7 @@ public class extends VectorExpression { int[] sel = batch.selected; boolean[] inputIsNull = inputColVector.isNull; boolean[] outputIsNull = outputColVector.isNull; -; + HiveDecimalWritable[] vector = inputColVector.vector; HiveDecimalWritable[] outputVector = outputColVector.vector; @@ -165,6 +165,10 @@ public class extends VectorExpression { } } + public HiveDecimal getValue() { + return value; + } + @Override public String vectorExpressionParameters() { return getColumnParamString(0, colNum) + ", val " + value.toString(); diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/ConstantVectorExpression.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/ConstantVectorExpression.java index 0a16e08d61..29c0454ab1 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/ConstantVectorExpression.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/ConstantVectorExpression.java @@ -438,6 +438,25 @@ public void setIntervalDayTimeValue(HiveIntervalDayTime intervalDayTimeValue) { public HiveIntervalDayTime getIntervalDayTimeValue() { return intervalDayTimeValue; } + + public Object getValue() { + switch (type) { + case LONG: + return getLongValue(); + case DOUBLE: + return getDoubleValue(); + case BYTES: + return getBytesValue(); + case DECIMAL: + return getDecimalValue(); + case TIMESTAMP: + return getTimestampValue(); + case INTERVAL_DAY_TIME: + return getIntervalDayTimeValue(); + default: + throw new RuntimeException("Unexpected column vector type " + type); + } + } public void setStructValue(Object structValue) throws HiveException { StructTypeInfo structTypeInfo = (StructTypeInfo) outputTypeInfo; diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java index c623adfb4d..1272b363f5 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java @@ -41,7 +41,10 @@ import org.apache.commons.lang.ArrayUtils; import org.apache.commons.lang3.tuple.ImmutablePair; +import org.apache.hadoop.hive.common.type.HiveDecimal; import org.apache.hadoop.hive.ql.exec.vector.VectorizedInputFormatInterface; +import org.apache.hadoop.hive.ql.exec.vector.expressions.ConstantVectorExpression; +import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.DecimalColDivideDecimalScalar; import org.apache.hadoop.hive.ql.exec.vector.reducesink.*; import org.apache.hadoop.hive.ql.exec.vector.udf.VectorUDFArgDesc; import org.apache.hadoop.hive.ql.io.AcidUtils; @@ -243,6 +246,8 @@ import com.google.common.collect.ImmutableSet; import com.google.common.base.Preconditions; +import javax.swing.*; + public class Vectorizer implements PhysicalPlanResolver { protected static transient final Logger LOG = LoggerFactory.getLogger(Vectorizer.class); @@ -4779,9 +4784,14 @@ private static VectorExpression fixDecimalDataTypePhysicalVariations(final Vecto } } } else { + Object[] arguments; int argumentCount = children.length + (parent.getOutputColumnNum() == -1 ? 0 : 1); - Object[] arguments = new Object[argumentCount]; - // new input column numbers + if (parent instanceof DecimalColDivideDecimalScalar) { + arguments = new Object[argumentCount + 1]; + arguments[children.length] = ((DecimalColDivideDecimalScalar) parent).getValue(); + } else { + arguments = new Object[argumentCount]; + } for (int i = 0; i < children.length; i++) { VectorExpression vce = children[i]; arguments[i] = vce.getOutputColumnNum(); diff --git ql/src/test/queries/clientpositive/vector_decimal_col_scalar_division.q ql/src/test/queries/clientpositive/vector_decimal_col_scalar_division.q new file mode 100644 index 0000000000..f2e67dfafe --- /dev/null +++ ql/src/test/queries/clientpositive/vector_decimal_col_scalar_division.q @@ -0,0 +1,20 @@ +CREATE temporary TABLE `catalog_Sales`( + `cs_quantity` int, + `cs_wholesale_cost` decimal(7,2), + `cs_list_price` decimal(7,2), + `cs_sales_price` decimal(7,2), + `cs_ext_discount_amt` decimal(7,2), + `cs_ext_sales_price` decimal(7,2), + `cs_ext_wholesale_cost` decimal(7,2), + `cs_ext_list_price` decimal(7,2), + `cs_ext_tax` decimal(7,2), + `cs_coupon_amt` decimal(7,2), + `cs_ext_ship_cost` decimal(7,2), + `cs_net_paid` decimal(7,2), + `cs_net_paid_inc_tax` decimal(7,2), + `cs_net_paid_inc_ship` decimal(7,2), + `cs_net_paid_inc_ship_tax` decimal(7,2), + `cs_net_profit` decimal(7,2)) + ; + +explain vectorization detail select max((((cs_ext_list_price - cs_ext_wholesale_cost) - cs_ext_discount_amt) + cs_ext_sales_price) / 2) from catalog_sales; diff --git ql/src/test/results/clientpositive/vector_decimal_col_scalar_division.q.out ql/src/test/results/clientpositive/vector_decimal_col_scalar_division.q.out new file mode 100644 index 0000000000..7417c1348b --- /dev/null +++ ql/src/test/results/clientpositive/vector_decimal_col_scalar_division.q.out @@ -0,0 +1,136 @@ +PREHOOK: query: CREATE temporary TABLE `catalog_Sales`( + `cs_quantity` int, + `cs_wholesale_cost` decimal(7,2), + `cs_list_price` decimal(7,2), + `cs_sales_price` decimal(7,2), + `cs_ext_discount_amt` decimal(7,2), + `cs_ext_sales_price` decimal(7,2), + `cs_ext_wholesale_cost` decimal(7,2), + `cs_ext_list_price` decimal(7,2), + `cs_ext_tax` decimal(7,2), + `cs_coupon_amt` decimal(7,2), + `cs_ext_ship_cost` decimal(7,2), + `cs_net_paid` decimal(7,2), + `cs_net_paid_inc_tax` decimal(7,2), + `cs_net_paid_inc_ship` decimal(7,2), + `cs_net_paid_inc_ship_tax` decimal(7,2), + `cs_net_profit` decimal(7,2)) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@catalog_Sales +POSTHOOK: query: CREATE temporary TABLE `catalog_Sales`( + `cs_quantity` int, + `cs_wholesale_cost` decimal(7,2), + `cs_list_price` decimal(7,2), + `cs_sales_price` decimal(7,2), + `cs_ext_discount_amt` decimal(7,2), + `cs_ext_sales_price` decimal(7,2), + `cs_ext_wholesale_cost` decimal(7,2), + `cs_ext_list_price` decimal(7,2), + `cs_ext_tax` decimal(7,2), + `cs_coupon_amt` decimal(7,2), + `cs_ext_ship_cost` decimal(7,2), + `cs_net_paid` decimal(7,2), + `cs_net_paid_inc_tax` decimal(7,2), + `cs_net_paid_inc_ship` decimal(7,2), + `cs_net_paid_inc_ship_tax` decimal(7,2), + `cs_net_profit` decimal(7,2)) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@catalog_Sales +PREHOOK: query: explain vectorization detail select max((((cs_ext_list_price - cs_ext_wholesale_cost) - cs_ext_discount_amt) + cs_ext_sales_price) / 2) from catalog_sales +PREHOOK: type: QUERY +PREHOOK: Input: default@catalog_sales +#### A masked pattern was here #### +POSTHOOK: query: explain vectorization detail select max((((cs_ext_list_price - cs_ext_wholesale_cost) - cs_ext_discount_amt) + cs_ext_sales_price) / 2) from catalog_sales +POSTHOOK: type: QUERY +POSTHOOK: Input: default@catalog_sales +#### A masked pattern was here #### +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: catalog_sales + Statistics: Num rows: 1 Data size: 448 Basic stats: COMPLETE Column stats: NONE + TableScan Vectorization: + native: true + vectorizationSchemaColumns: [0:cs_quantity:int, 1:cs_wholesale_cost:decimal(7,2)/DECIMAL_64, 2:cs_list_price:decimal(7,2)/DECIMAL_64, 3:cs_sales_price:decimal(7,2)/DECIMAL_64, 4:cs_ext_discount_amt:decimal(7,2)/DECIMAL_64, 5:cs_ext_sales_price:decimal(7,2)/DECIMAL_64, 6:cs_ext_wholesale_cost:decimal(7,2)/DECIMAL_64, 7:cs_ext_list_price:decimal(7,2)/DECIMAL_64, 8:cs_ext_tax:decimal(7,2)/DECIMAL_64, 9:cs_coupon_amt:decimal(7,2)/DECIMAL_64, 10:cs_ext_ship_cost:decimal(7,2)/DECIMAL_64, 11:cs_net_paid:decimal(7,2)/DECIMAL_64, 12:cs_net_paid_inc_tax:decimal(7,2)/DECIMAL_64, 13:cs_net_paid_inc_ship:decimal(7,2)/DECIMAL_64, 14:cs_net_paid_inc_ship_tax:decimal(7,2)/DECIMAL_64, 15:cs_net_profit:decimal(7,2)/DECIMAL_64, 16:ROW__ID:struct] + Select Operator + expressions: ((((cs_ext_list_price - cs_ext_wholesale_cost) - cs_ext_discount_amt) + cs_ext_sales_price) / 2) (type: decimal(14,6)) + outputColumnNames: _col0 + Select Vectorization: + className: VectorSelectOperator + native: true + projectedOutputColumnNums: [20] + selectExpressions: DecimalColDivideDecimalScalar(col 21:decimal(10,2), val 2)(children: ConvertDecimal64ToDecimal(col 19:decimal(10,2)/DECIMAL_64)(children: Decimal64ColAddDecimal64Column(col 18:decimal(9,2)/DECIMAL_64, col 5:decimal(7,2)/DECIMAL_64)(children: Decimal64ColSubtractDecimal64Column(col 17:decimal(8,2)/DECIMAL_64, col 4:decimal(7,2)/DECIMAL_64)(children: Decimal64ColSubtractDecimal64Column(col 7:decimal(7,2)/DECIMAL_64, col 6:decimal(7,2)/DECIMAL_64) -> 17:decimal(8,2)/DECIMAL_64) -> 18:decimal(9,2)/DECIMAL_64) -> 19:decimal(10,2)/DECIMAL_64) -> 21:decimal(10,2)) -> 20:decimal(14,6) + Statistics: Num rows: 1 Data size: 448 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: max(_col0) + Group By Vectorization: + aggregators: VectorUDAFMaxDecimal(col 20:decimal(14,6)) -> decimal(14,6) + className: VectorGroupByOperator + groupByMode: HASH + native: false + vectorProcessingMode: HASH + projectedOutputColumnNums: [0] + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 560 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Reduce Sink Vectorization: + className: VectorReduceSinkOperator + native: false + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + nativeConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false + Statistics: Num rows: 1 Data size: 560 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: decimal(14,6)) + Execution mode: vectorized + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vector.serde.deserialize IS true + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] + inputFileFormats: org.apache.hadoop.mapred.TextInputFormat + allNative: false + usesVectorUDFAdaptor: false + vectorized: true + rowBatchContext: + dataColumnCount: 16 + includeColumns: [4, 5, 6, 7] + dataColumns: cs_quantity:int, cs_wholesale_cost:decimal(7,2)/DECIMAL_64, cs_list_price:decimal(7,2)/DECIMAL_64, cs_sales_price:decimal(7,2)/DECIMAL_64, cs_ext_discount_amt:decimal(7,2)/DECIMAL_64, cs_ext_sales_price:decimal(7,2)/DECIMAL_64, cs_ext_wholesale_cost:decimal(7,2)/DECIMAL_64, cs_ext_list_price:decimal(7,2)/DECIMAL_64, cs_ext_tax:decimal(7,2)/DECIMAL_64, cs_coupon_amt:decimal(7,2)/DECIMAL_64, cs_ext_ship_cost:decimal(7,2)/DECIMAL_64, cs_net_paid:decimal(7,2)/DECIMAL_64, cs_net_paid_inc_tax:decimal(7,2)/DECIMAL_64, cs_net_paid_inc_ship:decimal(7,2)/DECIMAL_64, cs_net_paid_inc_ship_tax:decimal(7,2)/DECIMAL_64, cs_net_profit:decimal(7,2)/DECIMAL_64 + partitionColumnCount: 0 + scratchColumnTypeNames: [decimal(8,2)/DECIMAL_64, decimal(9,2)/DECIMAL_64, decimal(10,2)/DECIMAL_64, decimal(14,6), decimal(10,2)] + Reduce Vectorization: + enabled: false + enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true + enableConditionsNotMet: hive.execution.engine mr IN [tez, spark] IS false + Reduce Operator Tree: + Group By Operator + aggregations: max(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 560 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 560 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink +