diff --git a/data/files/decimal64table.csv b/data/files/decimal64table.csv index 1038ddfca6..d3fbccacaf 100644 --- a/data/files/decimal64table.csv +++ b/data/files/decimal64table.csv @@ -1,3 +1,4 @@ +null,null,null,null 1001,1,101,11 1002,2,102,12 1003,3,103,13 diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties index 608ec04fe7..935ba5cf1a 100644 --- a/itests/src/test/resources/testconfiguration.properties +++ b/itests/src/test/resources/testconfiguration.properties @@ -844,6 +844,7 @@ minillaplocal.query.files=\ vector_create_struct_table.q,\ vector_decimal_2.q,\ vector_decimal_udf.q,\ + vector_decimal64_case_when.q,\ vector_full_outer_join.q,\ vector_fullouter_mapjoin_1_fast.q,\ vector_fullouter_mapjoin_1_optimized.q,\ diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java index 627165d5be..0c09c86e93 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java @@ -36,6 +36,7 @@ import org.apache.commons.lang3.ArrayUtils; import org.apache.hadoop.hive.common.type.Date; +import org.apache.hadoop.hive.metastore.api.Decimal; import org.apache.hadoop.hive.ql.exec.vector.expressions.BucketNumExpression; import org.apache.hadoop.hive.ql.exec.vector.expressions.CastBooleanToCharViaLongToChar; import org.apache.hadoop.hive.ql.exec.vector.expressions.CastBooleanToStringViaLongToString; @@ -2473,14 +2474,35 @@ private VectorExpression getCoalesceExpression(List childExpr, final int size = vectorChildren.length; TypeInfo[] inputTypeInfos = new TypeInfo[size]; DataTypePhysicalVariation[] inputDataTypePhysicalVariations = new DataTypePhysicalVariation[size]; - int i = 0; - for (VectorExpression ve : vectorChildren) { + DataTypePhysicalVariation outputDataTypePhysicalVariation = DataTypePhysicalVariation.DECIMAL_64; + boolean fixConstants = false; + for (int i = 0; i < vectorChildren.length; ++i) { + VectorExpression ve = vectorChildren[i]; inputColumns[i] = ve.getOutputColumnNum(); inputTypeInfos[i] = ve.getOutputTypeInfo(); - inputDataTypePhysicalVariations[i++] = ve.getOutputDataTypePhysicalVariation(); + inputDataTypePhysicalVariations[i] = ve.getOutputDataTypePhysicalVariation(); + if (inputDataTypePhysicalVariations[i] == DataTypePhysicalVariation.NONE) { + if (childExpr.get(i) instanceof ExprNodeConstantDesc && inputTypeInfos[i] instanceof DecimalTypeInfo && + ((DecimalTypeInfo)inputTypeInfos[i]).precision() <= 18) { + fixConstants = true; + } else { + outputDataTypePhysicalVariation = DataTypePhysicalVariation.NONE; + } + } } - final int outputColumnNum = ocm.allocateOutputColumn(returnType); + if (outputDataTypePhysicalVariation == DataTypePhysicalVariation.DECIMAL_64 && fixConstants) { + for (int i = 0; i < vectorChildren.length; ++i) { + if (inputDataTypePhysicalVariations[i] == DataTypePhysicalVariation.NONE && + childExpr.get(i) instanceof ExprNodeConstantDesc) { + vectorChildren[i].setOutputDataTypePhysicalVariation(DataTypePhysicalVariation.DECIMAL_64); + int scratchColIndex = vectorChildren[i].getOutputColumnNum() - ocm.initialOutputCol; + ocm.scratchDataTypePhysicalVariations[scratchColIndex] = DataTypePhysicalVariation.DECIMAL_64; + } + } + } + + final int outputColumnNum = ocm.allocateOutputColumn(returnType, outputDataTypePhysicalVariation); VectorCoalesce vectorCoalesce = new VectorCoalesce(inputColumns, outputColumnNum); vectorCoalesce.setChildExpressions(vectorChildren); @@ -2489,7 +2511,7 @@ private VectorExpression getCoalesceExpression(List childExpr, vectorCoalesce.setInputDataTypePhysicalVariations(inputDataTypePhysicalVariations); vectorCoalesce.setOutputTypeInfo(returnType); - vectorCoalesce.setOutputDataTypePhysicalVariation(DataTypePhysicalVariation.NONE); + vectorCoalesce.setOutputDataTypePhysicalVariation(outputDataTypePhysicalVariation); freeNonColumns(vectorChildren); @@ -3877,8 +3899,9 @@ private VectorExpression doGetIfExpression(GenericUDFIf genericUDFIf, List] + Select Operator + expressions: ss_ext_list_price (type: decimal(7,2)) + outputColumnNames: ss_ext_list_price + Select Vectorization: + className: VectorSelectOperator + native: true + projectedOutputColumnNums: [0] + Statistics: Num rows: 1001 Data size: 112112 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: sum(COALESCE(ss_ext_list_price,1)) + Group By Vectorization: + aggregators: VectorUDAFSumDecimal64(VectorCoalesce(columns [0, 5])(children: col 0:decimal(7,2)/DECIMAL_64, ConstantVectorExpression(val 1) -> 5:decimal(7,2)/DECIMAL_64) -> 6:decimal(7,2)/DECIMAL_64) -> decimal(17,2)/DECIMAL_64 + className: VectorGroupByOperator + groupByMode: HASH + native: false + vectorProcessingMode: HASH + projectedOutputColumnNums: [0] + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Reduce Sink Vectorization: + className: VectorReduceSinkEmptyKeyOperator + native: true + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + valueColumns: 0:decimal(17,2) + Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: decimal(17,2)) + Execution mode: vectorized, llap + LLAP IO: all inputs + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] + inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat + allNative: false + usesVectorUDFAdaptor: false + vectorized: true + rowBatchContext: + dataColumnCount: 4 + includeColumns: [0] + dataColumns: ss_ext_list_price:decimal(7,2)/DECIMAL_64, ss_ext_wholesale_cost:decimal(19,1), ss_ext_discount_amt:int, ss_ext_sales_price:decimal(7,2)/DECIMAL_64 + partitionColumnCount: 0 + scratchColumnTypeNames: [decimal(7,2)/DECIMAL_64, decimal(7,2)/DECIMAL_64] + Reducer 2 + Execution mode: vectorized, llap + Reduce Vectorization: + enabled: true + enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true + reduceColumnNullOrder: + reduceColumnSortOrder: + allNative: false + usesVectorUDFAdaptor: false + vectorized: true + rowBatchContext: + dataColumnCount: 1 + dataColumns: VALUE._col0:decimal(17,2) + partitionColumnCount: 0 + scratchColumnTypeNames: [] + Reduce Operator Tree: + Group By Operator + aggregations: sum(VALUE._col0) + Group By Vectorization: + aggregators: VectorUDAFSumDecimal(col 0:decimal(17,2)) -> decimal(17,2) + className: VectorGroupByOperator + groupByMode: MERGEPARTIAL + native: false + vectorProcessingMode: GLOBAL + projectedOutputColumnNums: [0] + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + File Sink Vectorization: + className: VectorFileSinkOperator + native: false + Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select sum(NVL(ss_ext_list_price, 1.0)) from vector_decimal64_case_when_tmp +PREHOOK: type: QUERY +PREHOOK: Input: default@vector_decimal64_case_when_tmp +#### A masked pattern was here #### +POSTHOOK: query: select sum(NVL(ss_ext_list_price, 1.0)) from vector_decimal64_case_when_tmp +POSTHOOK: type: QUERY +POSTHOOK: Input: default@vector_decimal64_case_when_tmp +#### A masked pattern was here #### +1500500.00