diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java index a7b9e62cfd..086ec29b10 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java @@ -1763,8 +1763,7 @@ public RelNode apply(RelOptCluster cluster, RelOptSchema relOptSchema, SchemaPlu HiveRelFieldTrimmer fieldTrimmer = new HiveRelFieldTrimmer(null, HiveRelFactories.HIVE_BUILDER.create(optCluster, null), this.columnAccessInfo, this.viewProjectToTableSchema); - - fieldTrimmer.trim(calciteGenPlan); + calciteGenPlan = fieldTrimmer.trim(calciteGenPlan); // Create and set MD provider HiveDefaultRelMetadataProvider mdProvider = new HiveDefaultRelMetadataProvider(conf); diff --git a/ql/src/test/queries/clientpositive/ppd_windowing3.q b/ql/src/test/queries/clientpositive/ppd_windowing3.q new file mode 100644 index 0000000000..e7a2d4ff16 --- /dev/null +++ b/ql/src/test/queries/clientpositive/ppd_windowing3.q @@ -0,0 +1,73 @@ +DROP TABLE IF EXISTS t PURGE; +CREATE TABLE t ( + c1 string, + c2 INT, + c3 string, + c4 string +) +stored AS orc ; +INSERT INTO t VALUES +('1',0, '1', '2016-01-01'), +('2',1, '1', '2016-01-01'), +('3',2, '2', '2016-01-02'), +('4',3, '2', '2016-01-02'), +('1',0, '1', '2016-01-01'), +('2',1, '1', '2016-01-01'), +('3',2, '2', '2016-01-02'), +('4',3, '2', '2016-01-02'), +('1',0, '1', '2016-01-01'), +('2',2, '1', '2016-01-01'), +('3',4, '2', '2016-01-02'), +('4',6, '2', '2016-01-02'), +('1',0, '1', '2016-01-01'), +('2',1, '1', '2016-01-01'), +('3',2, '2', '2016-01-02'), +('4',3, '2', '2016-01-02'), +('1',0, '1', '2016-01-01'), +('2',2, '1', '2016-01-01'), +('3',4, '2', '2016-01-02'), +('4',6, '2', '2016-01-02'); +ANALYZE TABLE t COMPUTE STATISTICS FOR COLUMNS; + +DROP TABLE IF EXISTS u PURGE ; +CREATE TABLE u ( + d1 string, + d2 INT, + d3 string, + d4 string +) +stored AS orc ; + +INSERT INTO u +VALUES('1', 0, '1', '2016-01-01'), + ('2', 1, '1', '2016-01-01'), + ('3', 2, '2', '2016-01-02'), + ('4', 3, '2', '2016-01-02'); +INSERT INTO U +VALUES('1', 4, '1', '2016-01-01'), + ('2', 5, '1', '2016-01-01'), + ('3', 6, '2', '2016-01-02'), + ('4', 7, '2', '2016-01-02'); +ANALYZE TABLE u COMPUTE STATISTICS FOR COLUMNS; + +EXPLAIN +SELECT * +FROM +( + SELECT c1,c2,c3,c4,d1,d2,d3,d4 + FROM + ( + SELECT * , ROW_NUMBER() OVER(PARTITION BY d1 ORDER BY d2 DESC) AS rwnbU + FROM + ( + SELECT * + FROM ( + SELECT *, ROW_NUMBER() OVER(PARTITION BY c1, c2 ORDER BY c3 DESC) AS rwnbT + FROM t + ) AS T1 + ) AS T2 + INNER JOIN u ON d1=c1 + ) AS T3 +) AS T4 +WHERE c1 = 4 AND d2 = 7 ; + diff --git a/ql/src/test/results/clientpositive/ppd_windowing3.q.out b/ql/src/test/results/clientpositive/ppd_windowing3.q.out new file mode 100644 index 0000000000..28632f3e19 --- /dev/null +++ b/ql/src/test/results/clientpositive/ppd_windowing3.q.out @@ -0,0 +1,255 @@ +PREHOOK: query: DROP TABLE IF EXISTS t PURGE +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE IF EXISTS t PURGE +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE t ( + c1 string, + c2 INT, + c3 string, + c4 string +) +stored AS orc +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@t +POSTHOOK: query: CREATE TABLE t ( + c1 string, + c2 INT, + c3 string, + c4 string +) +stored AS orc +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@t +PREHOOK: query: INSERT INTO t VALUES +('1',0, '1', '2016-01-01'), +('2',1, '1', '2016-01-01'), +('3',2, '2', '2016-01-02'), +('4',3, '2', '2016-01-02'), +('1',0, '1', '2016-01-01'), +('2',1, '1', '2016-01-01'), +('3',2, '2', '2016-01-02'), +('4',3, '2', '2016-01-02'), +('1',0, '1', '2016-01-01'), +('2',2, '1', '2016-01-01'), +('3',4, '2', '2016-01-02'), +('4',6, '2', '2016-01-02'), +('1',0, '1', '2016-01-01'), +('2',1, '1', '2016-01-01'), +('3',2, '2', '2016-01-02'), +('4',3, '2', '2016-01-02'), +('1',0, '1', '2016-01-01'), +('2',2, '1', '2016-01-01'), +('3',4, '2', '2016-01-02'), +('4',6, '2', '2016-01-02') +PREHOOK: type: QUERY +PREHOOK: Output: default@t +POSTHOOK: query: INSERT INTO t VALUES +('1',0, '1', '2016-01-01'), +('2',1, '1', '2016-01-01'), +('3',2, '2', '2016-01-02'), +('4',3, '2', '2016-01-02'), +('1',0, '1', '2016-01-01'), +('2',1, '1', '2016-01-01'), +('3',2, '2', '2016-01-02'), +('4',3, '2', '2016-01-02'), +('1',0, '1', '2016-01-01'), +('2',2, '1', '2016-01-01'), +('3',4, '2', '2016-01-02'), +('4',6, '2', '2016-01-02'), +('1',0, '1', '2016-01-01'), +('2',1, '1', '2016-01-01'), +('3',2, '2', '2016-01-02'), +('4',3, '2', '2016-01-02'), +('1',0, '1', '2016-01-01'), +('2',2, '1', '2016-01-01'), +('3',4, '2', '2016-01-02'), +('4',6, '2', '2016-01-02') +POSTHOOK: type: QUERY +POSTHOOK: Output: default@t +POSTHOOK: Lineage: t.c1 SIMPLE [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +POSTHOOK: Lineage: t.c2 EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col2, type:string, comment:), ] +POSTHOOK: Lineage: t.c3 SIMPLE [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col3, type:string, comment:), ] +POSTHOOK: Lineage: t.c4 SIMPLE [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col4, type:string, comment:), ] +PREHOOK: query: ANALYZE TABLE t COMPUTE STATISTICS FOR COLUMNS +PREHOOK: type: QUERY +PREHOOK: Input: default@t +#### A masked pattern was here #### +POSTHOOK: query: ANALYZE TABLE t COMPUTE STATISTICS FOR COLUMNS +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t +#### A masked pattern was here #### +PREHOOK: query: DROP TABLE IF EXISTS u PURGE +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE IF EXISTS u PURGE +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE u ( + d1 string, + d2 INT, + d3 string, + d4 string +) +stored AS orc +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@u +POSTHOOK: query: CREATE TABLE u ( + d1 string, + d2 INT, + d3 string, + d4 string +) +stored AS orc +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@u +PREHOOK: query: INSERT INTO u +VALUES('1', 0, '1', '2016-01-01'), + ('2', 1, '1', '2016-01-01'), + ('3', 2, '2', '2016-01-02'), + ('4', 3, '2', '2016-01-02') +PREHOOK: type: QUERY +PREHOOK: Output: default@u +POSTHOOK: query: INSERT INTO u +VALUES('1', 0, '1', '2016-01-01'), + ('2', 1, '1', '2016-01-01'), + ('3', 2, '2', '2016-01-02'), + ('4', 3, '2', '2016-01-02') +POSTHOOK: type: QUERY +POSTHOOK: Output: default@u +POSTHOOK: Lineage: u.d1 SIMPLE [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +POSTHOOK: Lineage: u.d2 EXPRESSION [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col2, type:string, comment:), ] +POSTHOOK: Lineage: u.d3 SIMPLE [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col3, type:string, comment:), ] +POSTHOOK: Lineage: u.d4 SIMPLE [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col4, type:string, comment:), ] +PREHOOK: query: INSERT INTO U +VALUES('1', 4, '1', '2016-01-01'), + ('2', 5, '1', '2016-01-01'), + ('3', 6, '2', '2016-01-02'), + ('4', 7, '2', '2016-01-02') +PREHOOK: type: QUERY +PREHOOK: Output: default@u +POSTHOOK: query: INSERT INTO U +VALUES('1', 4, '1', '2016-01-01'), + ('2', 5, '1', '2016-01-01'), + ('3', 6, '2', '2016-01-02'), + ('4', 7, '2', '2016-01-02') +POSTHOOK: type: QUERY +POSTHOOK: Output: default@u +POSTHOOK: Lineage: u.d1 SIMPLE [(values__tmp__table__3)values__tmp__table__3.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +POSTHOOK: Lineage: u.d2 EXPRESSION [(values__tmp__table__3)values__tmp__table__3.FieldSchema(name:tmp_values_col2, type:string, comment:), ] +POSTHOOK: Lineage: u.d3 SIMPLE [(values__tmp__table__3)values__tmp__table__3.FieldSchema(name:tmp_values_col3, type:string, comment:), ] +POSTHOOK: Lineage: u.d4 SIMPLE [(values__tmp__table__3)values__tmp__table__3.FieldSchema(name:tmp_values_col4, type:string, comment:), ] +PREHOOK: query: ANALYZE TABLE u COMPUTE STATISTICS FOR COLUMNS +PREHOOK: type: QUERY +PREHOOK: Input: default@u +#### A masked pattern was here #### +POSTHOOK: query: ANALYZE TABLE u COMPUTE STATISTICS FOR COLUMNS +POSTHOOK: type: QUERY +POSTHOOK: Input: default@u +#### A masked pattern was here #### +PREHOOK: query: EXPLAIN +SELECT * +FROM +( + SELECT c1,c2,c3,c4,d1,d2,d3,d4 + FROM + ( + SELECT * , ROW_NUMBER() OVER(PARTITION BY d1 ORDER BY d2 DESC) AS rwnbU + FROM + ( + SELECT * + FROM ( + SELECT *, ROW_NUMBER() OVER(PARTITION BY c1, c2 ORDER BY c3 DESC) AS rwnbT + FROM t + ) AS T1 + ) AS T2 + INNER JOIN u ON d1=c1 + ) AS T3 +) AS T4 +WHERE c1 = 4 AND d2 = 7 +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN +SELECT * +FROM +( + SELECT c1,c2,c3,c4,d1,d2,d3,d4 + FROM + ( + SELECT * , ROW_NUMBER() OVER(PARTITION BY d1 ORDER BY d2 DESC) AS rwnbU + FROM + ( + SELECT * + FROM ( + SELECT *, ROW_NUMBER() OVER(PARTITION BY c1, c2 ORDER BY c3 DESC) AS rwnbT + FROM t + ) AS T1 + ) AS T2 + INNER JOIN u ON d1=c1 + ) AS T3 +) AS T4 +WHERE c1 = 4 AND d2 = 7 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: t + Statistics: Num rows: 20 Data size: 5360 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (UDFToDouble(c1) = 4.0) (type: boolean) + Statistics: Num rows: 10 Data size: 2680 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: c1 (type: string), c2 (type: int), c3 (type: string), c4 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 10 Data size: 2680 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 10 Data size: 2680 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: int), _col2 (type: string), _col3 (type: string) + TableScan + alias: u + Statistics: Num rows: 8 Data size: 2144 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((UDFToDouble(d1) = 4.0) and (d2 = 7)) (type: boolean) + Statistics: Num rows: 2 Data size: 536 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: d1 (type: string), 7 (type: int), d3 (type: string), d4 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 2 Data size: 536 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 2 Data size: 536 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: int), _col2 (type: string), _col3 (type: string) + Reduce Operator Tree: + Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + Statistics: Num rows: 11 Data size: 2948 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 11 Data size: 2948 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink +