diff --git data/files/studentnull100 data/files/studentnull100 new file mode 100644 index 0000000..6140c8a --- /dev/null +++ data/files/studentnull100 @@ -0,0 +1,100 @@ +tom thompson420.53 +luke king280.47 +priscilla falkner551.16 +luke brown601.14 +ulysses garcia352.74 +calvin brown282.70 +oscar thompson352.98 +xavier garcia331.06 +nick johnson34 +quinn ovid19 +653.45 +fred polk340.23 +victor van buren751.64 +yuri laertes353.74 +rachel zipper322.02 +victor laertes651.51 +563.95 +sarah ovid1.62 +jessica king722.08 +bob falkner660.16 +jessica brown3.80 +nick underhill423.41 +ethan carson410.85 +mike xylophone510.00 +nick thompson343.95 +david king473.41 +yuri allen373.98 +yuri robinson261.22 +xavier laertes510.51 +fred falkner512.33 +alice king473.88 +quinn ellison443.62 +luke white482.14 +742.22 +ulysses miller243.97 +mike allen623.80 +bob ovid693.12 +david garcia451.13 +luke brown752.84 +tom robinson292.79 +gabriella quirinius573.83 +gabriella miller240.76 +oscar xylophone440.63 +oscar zipper372.85 +ulysses king761.84 +priscilla ovid691.43 +oscar ichabod392.41 +luke young301.85 +victor falkner63 +holly ovid281.68 +quinn hernandez562.12 +gabriella young473.12 +tom garcia502.13 +531.51 +quinn ovid463.87 +wendy van buren64 +katie king472.93 +oscar miller443.33 +nick ellison571.64 +wendy quirinius192.31 +xavier robinson322.78 +rachel ichabod731.72 +sarah ichabod300.11 +xavier underhill712.28 +yuri carson371.55 +alice van buren233.55 +mike robinson592.60 +holly johnson552.85 +mike white37 +wendy ellison200.55 +jessica king282.92 +453.53 +rachel polk313.60 +david laertes570.39 +priscilla underhill551.70 +bob laertes732.93 +ulysses miller701.16 +quinn ovid341.84 +holly underhill281.13 +xavier xylophone502.06 +463.38 +victor van buren761.39 +irene zipper261.08 +zach robinson560.86 +wendy king36 +irene polk763.81 +sarah polk491.47 +jessica underhill183.51 +gabriella van buren763.98 +yuri zipper69 +ethan young183.61 +nick hernandez223.30 +holly steinbeck453.36 +holly carson611.38 +david robinson672.37 +ulysses falkner553.70 +fred nixon440.62 +holly quirinius240.71 +victor ovid663.88 +bob robinson412.13 diff --git itests/src/test/resources/testconfiguration.properties itests/src/test/resources/testconfiguration.properties index 1ce3ba6..a277cb7 100644 --- itests/src/test/resources/testconfiguration.properties +++ itests/src/test/resources/testconfiguration.properties @@ -730,6 +730,7 @@ minillaplocal.query.files=\ vector_mapjoin_reduce.q,\ vector_number_compare_projection.q,\ vector_partitioned_date_time.q,\ + vector_ptf_1.q,\ vector_ptf_part_simple.q,\ vector_udf_adaptor_1,\ vector_udf1.q,\ diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java index 99618c6..190771e 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java @@ -2442,6 +2442,26 @@ private boolean validatePTFOperator(PTFOperator op, VectorizationContext vContex setOperatorIssue("PTF Mapper not supported"); return false; } + List> ptfParents = op.getParentOperators(); + if (ptfParents != null && ptfParents.size() > 0) { + Operator ptfParent = op.getParentOperators().get(0); + if (!(ptfParent instanceof ReduceSinkOperator)) { + boolean isReduceShufflePtf = false; + if (ptfParent instanceof SelectOperator) { + ptfParents = ptfParent.getParentOperators(); + if (ptfParents == null || ptfParents.size() == 0) { + isReduceShufflePtf = true; + } else { + ptfParent = ptfParent.getParentOperators().get(0); + isReduceShufflePtf = (ptfParent instanceof ReduceSinkOperator); + } + } + if (!isReduceShufflePtf) { + setOperatorIssue("Only PTF directly under reduce-shuffle is supported"); + return false; + } + } + } boolean forNoop = ptfDesc.forNoop(); if (forNoop) { setOperatorIssue("NOOP not supported"); diff --git ql/src/test/queries/clientpositive/vector_ptf_1.q ql/src/test/queries/clientpositive/vector_ptf_1.q new file mode 100644 index 0000000..9bb920e --- /dev/null +++ ql/src/test/queries/clientpositive/vector_ptf_1.q @@ -0,0 +1,23 @@ +set hive.cli.print.header=true; +set hive.mapred.mode=nonstrict; +set hive.explain.user=false; +set hive.vectorized.execution.enabled=true; + +create table studentnull100( +name string, +age int, +gpa double) +row format delimited +fields terminated by '\001' +stored as textfile; +LOAD DATA LOCAL INPATH '../../data/files/studentnull100' OVERWRITE INTO TABLE studentnull100; +analyze table studentnull100 compute statistics; + +explain vectorization detail +select age, name, avg(gpa), sum(age) over (partition by name) +from studentnull100 +group by age, name; + +select age, name, avg(gpa), sum(age) over (partition by name) +from studentnull100 +group by age, name; \ No newline at end of file diff --git ql/src/test/results/clientpositive/llap/vector_ptf_1.q.out ql/src/test/results/clientpositive/llap/vector_ptf_1.q.out new file mode 100644 index 0000000..df0dd21 --- /dev/null +++ ql/src/test/results/clientpositive/llap/vector_ptf_1.q.out @@ -0,0 +1,294 @@ +PREHOOK: query: create table studentnull100( +name string, +age int, +gpa double) +row format delimited +fields terminated by '\001' +stored as textfile +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@studentnull100 +POSTHOOK: query: create table studentnull100( +name string, +age int, +gpa double) +row format delimited +fields terminated by '\001' +stored as textfile +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@studentnull100 +PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/studentnull100' OVERWRITE INTO TABLE studentnull100 +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@studentnull100 +POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/studentnull100' OVERWRITE INTO TABLE studentnull100 +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@studentnull100 +PREHOOK: query: analyze table studentnull100 compute statistics +PREHOOK: type: QUERY +PREHOOK: Input: default@studentnull100 +PREHOOK: Output: default@studentnull100 +POSTHOOK: query: analyze table studentnull100 compute statistics +POSTHOOK: type: QUERY +POSTHOOK: Input: default@studentnull100 +POSTHOOK: Output: default@studentnull100 +studentnull100.name studentnull100.age studentnull100.gpa +PREHOOK: query: explain vectorization detail +select age, name, avg(gpa), sum(age) over (partition by name) +from studentnull100 +group by age, name +PREHOOK: type: QUERY +POSTHOOK: query: explain vectorization detail +select age, name, avg(gpa), sum(age) over (partition by name) +from studentnull100 +group by age, name +POSTHOOK: type: QUERY +Explain +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: studentnull100 + Statistics: Num rows: 100 Data size: 18816 Basic stats: COMPLETE Column stats: NONE + TableScan Vectorization: + native: true + vectorizationSchemaColumns: [0:name:string, 1:age:int, 2:gpa:double, 3:ROW__ID:struct] + Select Operator + expressions: name (type: string), age (type: int), gpa (type: double) + outputColumnNames: name, age, gpa + Select Vectorization: + className: VectorSelectOperator + native: true + projectedOutputColumnNums: [0, 1, 2] + Statistics: Num rows: 100 Data size: 18816 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: avg(gpa) + Group By Vectorization: + aggregators: VectorUDAFAvgDouble(col 2:double) -> struct + className: VectorGroupByOperator + groupByMode: HASH + keyExpressions: col 0:string, col 1:int + native: false + vectorProcessingMode: HASH + projectedOutputColumnNums: [0] + keys: name (type: string), age (type: int) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 100 Data size: 18816 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: int) + sort order: ++ + Map-reduce partition columns: _col0 (type: string) + Reduce Sink Vectorization: + className: VectorReduceSinkObjectHashOperator + keyColumnNums: [0, 1] + native: true + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + partitionColumnNums: [0] + valueColumnNums: [2] + Statistics: Num rows: 100 Data size: 18816 Basic stats: COMPLETE Column stats: NONE + value expressions: _col2 (type: struct) + Execution mode: vectorized, llap + LLAP IO: no inputs + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vector.serde.deserialize IS true + inputFormatFeatureSupport: [DECIMAL_64] + vectorizationSupportRemovedReasons: [DECIMAL_64 disabled because LLAP is enabled] + featureSupportInUse: [] + inputFileFormats: org.apache.hadoop.mapred.TextInputFormat + allNative: false + usesVectorUDFAdaptor: false + vectorized: true + rowBatchContext: + dataColumnCount: 3 + includeColumns: [0, 1, 2] + dataColumns: name:string, age:int, gpa:double + partitionColumnCount: 0 + scratchColumnTypeNames: [] + Reducer 2 + Execution mode: llap + Reduce Vectorization: + enabled: true + enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true + notVectorizedReason: PTF operator: Only PTF directly under reduce-shuffle is supported + vectorized: false + Reduce Operator Tree: + Group By Operator + aggregations: avg(VALUE._col0) + keys: KEY._col0 (type: string), KEY._col1 (type: int) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 50 Data size: 9408 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col1 (type: int), _col0 (type: string), _col2 (type: double) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 50 Data size: 9408 Basic stats: COMPLETE Column stats: NONE + PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: int, _col1: string, _col2: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 ASC NULLS FIRST + partition by: _col1 + raw input shape: + window functions: + window function definition + alias: sum_window_0 + arguments: _col0 + name: sum + window function: GenericUDAFSumLong + window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) + Statistics: Num rows: 50 Data size: 9408 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: int), _col1 (type: string), _col2 (type: double), sum_window_0 (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 50 Data size: 9408 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 50 Data size: 9408 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select age, name, avg(gpa), sum(age) over (partition by name) +from studentnull100 +group by age, name +PREHOOK: type: QUERY +PREHOOK: Input: default@studentnull100 +#### A masked pattern was here #### +POSTHOOK: query: select age, name, avg(gpa), sum(age) over (partition by name) +from studentnull100 +group by age, name +POSTHOOK: type: QUERY +POSTHOOK: Input: default@studentnull100 +#### A masked pattern was here #### +age name _c2 sum_window_0 +45 3.53 339 +46 3.38 339 +53 1.51 339 +56 3.95 339 +65 3.45 339 +74 2.22 339 +47 alice king 3.88 47 +23 alice van buren 3.55 23 +66 bob falkner 0.16 66 +73 bob laertes 2.93 73 +69 bob ovid 3.12 69 +41 bob robinson 2.13 41 +28 calvin brown 2.7 28 +45 david garcia 1.13 45 +47 david king 3.41 47 +57 david laertes 0.39 57 +67 david robinson 2.37 67 +41 ethan carson 0.85 41 +18 ethan young 3.61 18 +51 fred falkner 2.33 51 +44 fred nixon 0.62 44 +34 fred polk 0.23 34 +24 gabriella miller 0.76 24 +57 gabriella quirinius 3.83 57 +76 gabriella van buren 3.98 76 +47 gabriella young 3.12 47 +61 holly carson 1.38 61 +55 holly johnson 2.85 55 +28 holly ovid 1.68 28 +24 holly quirinius 0.71 24 +45 holly steinbeck 3.36 45 +28 holly underhill 1.13 28 +76 irene polk 3.81 76 +26 irene zipper 1.08 26 +NULL jessica brown 3.8 NULL +28 jessica king 2.92 100 +72 jessica king 2.08 100 +18 jessica underhill 3.51 18 +47 katie king 2.93 47 +60 luke brown 1.14 135 +75 luke brown 2.84 135 +28 luke king 0.47 28 +48 luke white 2.14 48 +30 luke young 1.85 30 +62 mike allen 3.8 62 +59 mike robinson 2.6 59 +37 mike white NULL 37 +51 mike xylophone 0.0 51 +57 nick ellison 1.64 57 +22 nick hernandez 3.3 22 +34 nick johnson NULL 34 +34 nick thompson 3.95 34 +42 nick underhill 3.41 42 +39 oscar ichabod 2.41 39 +44 oscar miller 3.33 44 +35 oscar thompson 2.98 35 +44 oscar xylophone 0.63 44 +37 oscar zipper 2.85 37 +55 priscilla falkner 1.16 55 +69 priscilla ovid 1.43 69 +55 priscilla underhill 1.7 55 +44 quinn ellison 3.62 44 +56 quinn hernandez 2.12 56 +19 quinn ovid NULL 99 +34 quinn ovid 1.84 99 +46 quinn ovid 3.87 99 +73 rachel ichabod 1.72 73 +31 rachel polk 3.6 31 +32 rachel zipper 2.02 32 +30 sarah ichabod 0.11 30 +NULL sarah ovid 1.62 NULL +49 sarah polk 1.47 49 +50 tom garcia 2.13 50 +29 tom robinson 2.79 29 +42 tom thompson 0.53 42 +55 ulysses falkner 3.7 55 +35 ulysses garcia 2.74 35 +76 ulysses king 1.84 76 +24 ulysses miller 3.97 94 +70 ulysses miller 1.16 94 +63 victor falkner NULL 63 +65 victor laertes 1.51 65 +66 victor ovid 3.88 66 +75 victor van buren 1.64 151 +76 victor van buren 1.39 151 +20 wendy ellison 0.55 20 +36 wendy king NULL 36 +19 wendy quirinius 2.31 19 +64 wendy van buren NULL 64 +33 xavier garcia 1.06 33 +51 xavier laertes 0.51 51 +32 xavier robinson 2.78 32 +71 xavier underhill 2.28 71 +50 xavier xylophone 2.06 50 +37 yuri allen 3.98 37 +37 yuri carson 1.55 37 +35 yuri laertes 3.74 35 +26 yuri robinson 1.22 26 +69 yuri zipper NULL 69 +56 zach robinson 0.86 56 diff --git ql/src/test/results/clientpositive/llap/vector_windowing.q.out ql/src/test/results/clientpositive/llap/vector_windowing.q.out index 8dfee97..530610d 100644 --- ql/src/test/results/clientpositive/llap/vector_windowing.q.out +++ ql/src/test/results/clientpositive/llap/vector_windowing.q.out @@ -276,7 +276,7 @@ STAGE PLANS: Reduce Vectorization: enabled: true enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true - notVectorizedReason: PTF operator: lag not in supported functions [avg, count, dense_rank, first_value, last_value, max, min, rank, row_number, sum] + notVectorizedReason: PTF operator: Only PTF directly under reduce-shuffle is supported vectorized: false Reduce Operator Tree: Group By Operator @@ -484,7 +484,7 @@ STAGE PLANS: Reduce Vectorization: enabled: true enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true - notVectorizedReason: PTF operator: lag not in supported functions [avg, count, dense_rank, first_value, last_value, max, min, rank, row_number, sum] + notVectorizedReason: PTF operator: Only PTF directly under reduce-shuffle is supported vectorized: false Reduce Operator Tree: Group By Operator @@ -4553,31 +4553,15 @@ STAGE PLANS: partitionColumnCount: 0 scratchColumnTypeNames: [] Reducer 2 - Execution mode: vectorized, llap + Execution mode: llap Reduce Vectorization: enabled: true enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true - reduceColumnNullOrder: aa - reduceColumnSortOrder: ++ - allNative: false - usesVectorUDFAdaptor: false - vectorized: true - rowBatchContext: - dataColumnCount: 3 - dataColumns: KEY._col0:string, KEY._col1:string, VALUE._col0:double - partitionColumnCount: 0 - scratchColumnTypeNames: [] + notVectorizedReason: PTF operator: Only PTF directly under reduce-shuffle is supported + vectorized: false Reduce Operator Tree: Group By Operator aggregations: sum(VALUE._col0) - Group By Vectorization: - aggregators: VectorUDAFSumDouble(col 2:double) -> double - className: VectorGroupByOperator - groupByMode: MERGEPARTIAL - keyExpressions: col 0:string, col 1:string - native: false - vectorProcessingMode: MERGE_PARTIAL - projectedOutputColumnNums: [0] keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1, _col2 @@ -4601,37 +4585,14 @@ STAGE PLANS: name: sum window function: GenericUDAFSumDouble window frame: RANGE PRECEDING(MAX)~CURRENT - PTF Vectorization: - className: VectorPTFOperator - evaluatorClasses: [VectorPTFEvaluatorDoubleSum] - functionInputExpressions: [RoundWithNumDigitsDoubleToDouble(col 2, decimalPlaces 2) -> 4:double] - functionNames: [sum] - keyInputColumns: [0] - native: true - nonKeyInputColumns: [1, 2] - orderExpressions: [col 0:string] - outputColumns: [3, 0, 1, 2] - outputTypes: [double, string, string, double] - streamingColumns: [] Statistics: Num rows: 13 Data size: 2574 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col0 (type: string), _col1 (type: string), round(_col2, 2) (type: double), round(sum_window_0, 2) (type: double) outputColumnNames: _col0, _col1, _col2, _col3 - Select Vectorization: - className: VectorSelectOperator - native: true - projectedOutputColumnNums: [0, 1, 5, 6] - selectExpressions: RoundWithNumDigitsDoubleToDouble(col 2, decimalPlaces 2) -> 5:double, RoundWithNumDigitsDoubleToDouble(col 3, decimalPlaces 2) -> 6:double Statistics: Num rows: 13 Data size: 2678 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) sort order: ++ - Reduce Sink Vectorization: - className: VectorReduceSinkObjectHashOperator - keyColumnNums: [0, 1] - native: true - nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true - valueColumnNums: [5, 6] Statistics: Num rows: 13 Data size: 2678 Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col2 (type: double), _col3 (type: double) Reducer 3 @@ -4700,6 +4661,22 @@ POSTHOOK: Input: default@mfgr_price_view POSTHOOK: Input: default@part #### A masked pattern was here #### sq.p_mfgr sq.p_brand sq.s sq.s1 +Manufacturer#1 Brand#12 4800.84 8749.73 +Manufacturer#1 Brand#14 2346.3 8749.73 +Manufacturer#1 Brand#15 1602.59 8749.73 +Manufacturer#2 Brand#22 3491.38 8923.62 +Manufacturer#2 Brand#23 2031.98 8923.62 +Manufacturer#2 Brand#24 1698.66 8923.62 +Manufacturer#2 Brand#25 1701.6 8923.62 +Manufacturer#3 Brand#31 1671.68 7532.61 +Manufacturer#3 Brand#32 3333.37 7532.61 +Manufacturer#3 Brand#34 1337.29 7532.61 +Manufacturer#3 Brand#35 1190.27 7532.61 +Manufacturer#4 Brand#41 4755.94 7337.62 +Manufacturer#4 Brand#42 2581.68 7337.62 +Manufacturer#5 Brand#51 1611.66 7672.66 +Manufacturer#5 Brand#52 3254.17 7672.66 +Manufacturer#5 Brand#53 2806.83 7672.66 PREHOOK: query: select p_mfgr, p_brand, s, round(sum(s) over w1 ,2) as s1 from mfgr_price_view @@ -6134,7 +6111,7 @@ STAGE PLANS: Reduce Vectorization: enabled: true enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true - notVectorizedReason: PTF operator: lag not in supported functions [avg, count, dense_rank, first_value, last_value, max, min, rank, row_number, sum] + notVectorizedReason: PTF operator: Only PTF directly under reduce-shuffle is supported vectorized: false Reduce Operator Tree: Group By Operator diff --git ql/src/test/results/clientpositive/llap/vectorized_ptf.q.out ql/src/test/results/clientpositive/llap/vectorized_ptf.q.out index 9e06471..9ac1188 100644 --- ql/src/test/results/clientpositive/llap/vectorized_ptf.q.out +++ ql/src/test/results/clientpositive/llap/vectorized_ptf.q.out @@ -1297,7 +1297,7 @@ STAGE PLANS: Reduce Vectorization: enabled: true enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true - notVectorizedReason: PTF operator: lag not in supported functions [avg, count, dense_rank, first_value, last_value, max, min, rank, row_number, sum] + notVectorizedReason: PTF operator: Only PTF directly under reduce-shuffle is supported vectorized: false Reduce Operator Tree: Group By Operator @@ -3583,7 +3583,7 @@ STAGE PLANS: Reduce Vectorization: enabled: true enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true - notVectorizedReason: PTF operator: NOOP not supported + notVectorizedReason: PTF operator: Only PTF directly under reduce-shuffle is supported vectorized: false Reduce Operator Tree: Group By Operator diff --git ql/src/test/results/clientpositive/spark/vectorized_ptf.q.out ql/src/test/results/clientpositive/spark/vectorized_ptf.q.out index d91cb0e..8866890 100644 --- ql/src/test/results/clientpositive/spark/vectorized_ptf.q.out +++ ql/src/test/results/clientpositive/spark/vectorized_ptf.q.out @@ -1277,7 +1277,7 @@ STAGE PLANS: Reduce Vectorization: enabled: true enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine spark IN [tez, spark] IS true - notVectorizedReason: PTF operator: lag not in supported functions [avg, count, dense_rank, first_value, last_value, max, min, rank, row_number, sum] + notVectorizedReason: PTF operator: Only PTF directly under reduce-shuffle is supported vectorized: false Reduce Operator Tree: Group By Operator @@ -3533,7 +3533,7 @@ STAGE PLANS: Reduce Vectorization: enabled: true enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine spark IN [tez, spark] IS true - notVectorizedReason: PTF operator: NOOP not supported + notVectorizedReason: PTF operator: Only PTF directly under reduce-shuffle is supported vectorized: false Reduce Operator Tree: Group By Operator