################################################################### Source Table ################################################################### CREATE EXTERNAL TABLE analyst1.lineitem_partitioned( `l_orderkey` int, `l_partkey` int, `l_suppkey` int, `l_linenumber` int, `l_quantity` double, `l_extendedprice` double, `l_discount` double, `l_tax` double, `l_returnflag` string, `l_linestatus` string, `l_commitdate` string, `l_receiptdate` string, `l_shipinstruct` string, `l_shipmode` string, `l_comment` string) PARTITIONED BY ( `l_shipdate` string) STORED AS PARQUET LOCATION '/user/analyst1/tpch/sf100/lineitem'; ################################################################### Destination Table ################################################################### CREATE EXTERNAL TABLE analyst1.test1( l_commitdate string, l_receiptdate string PARTITIONED BY (`l_shipdate` string) STORED AS PARQUET LOCATION '/user/analyst1/tpch/sf100/lineitem_partitioned'; ################################################################### Query ################################################################### explain insert overwrite table analyst1.test1 select l_commitdate, l_receiptdate from analyst1.lineitem_partitioned where l_shipdate = '1992-01-02' group by l_commitdate, l_receiptdate; ################################################################### Below is when masking is enabled ################################################################### Explain STAGE DEPENDENCIES: Stage-1 is a root stage Stage-2 depends on stages: Stage-1 Stage-0 depends on stages: Stage-2 Stage-3 depends on stages: Stage-0 STAGE PLANS: Stage: Stage-1 Tez DagId: hive_20190310233213_3ca64c15-b953-463e-8eae-da8159d3ff85:14 Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) Reducer 3 <- Reducer 2 (SIMPLE_EDGE) DagName: Vertices: Map 1 Map Operator Tree: TableScan alias: lineitem_partitioned Statistics: Num rows: 329934278 Data size: 32993550885 Basic stats: COMPLETE Column stats: NONE GatherStats: false Select Operator expressions: l_receiptdate (type: string), l_shipdate (type: string) outputColumnNames: _col11, _col15 Statistics: Num rows: 329934278 Data size: 32993550885 Basic stats: COMPLETE Column stats: NONE Filter Operator isSamplingPred: false predicate: (_col15 = '1992-01-02') (type: boolean) Statistics: Num rows: 164967139 Data size: 16496775442 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: 'XXXXX' (type: string), _col11 (type: string), _col15 (type: string) mode: hash outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 164967139 Data size: 16496775442 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) null sort order: aaa sort order: +++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string), _col2 (type: string) Statistics: Num rows: 164967139 Data size: 16496775442 Basic stats: COMPLETE Column stats: NONE tag: -1 auto parallelism: true Path -> Alias: /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-01-02 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-01-03 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-01-04 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-01-05 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-01-06 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-01-07 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-01-08 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-01-09 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-01-10 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-01-11 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-01-12 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-01-13 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-01-14 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-01-15 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-01-16 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-01-17 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-01-18 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-01-19 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-01-20 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-01-21 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-01-22 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-01-23 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-01-24 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-01-25 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-01-26 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-01-27 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-01-28 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-01-29 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-01-30 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-01-31 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-02-01 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-02-02 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-02-03 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-02-04 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-02-05 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-02-06 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-02-07 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-02-08 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-02-09 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-02-10 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-02-11 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-02-12 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-02-13 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-02-14 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-02-15 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-02-16 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-02-17 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-02-18 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-02-19 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-02-20 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-02-21 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-02-22 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-02-23 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-02-24 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-02-25 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-02-26 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-02-27 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-02-28 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-02-29 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-03-01 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-03-02 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-03-03 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-03-04 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-03-05 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-03-06 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-03-07 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-03-08 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-03-09 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-03-10 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-03-11 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-03-12 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-03-13 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-03-14 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-03-15 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-03-16 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-03-17 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-03-18 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-03-19 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-03-20 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-03-21 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-03-22 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-03-23 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-03-24 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-03-25 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-03-26 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-03-27 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-03-28 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-03-29 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-03-30 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-03-31 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-04-01 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-04-02 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-04-03 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-04-04 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-04-05 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-04-06 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-04-07 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-04-08 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-04-09 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-04-10 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-04-11 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-04-12 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-04-13 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-04-14 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-04-15 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-04-16 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-04-17 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-04-18 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-04-19 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-04-20 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-04-21 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-04-22 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-04-23 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-04-24 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-04-25 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-04-26 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-04-27 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-04-28 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-04-29 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-04-30 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-05-01 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-05-02 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-05-03 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-05-04 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-05-05 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-05-06 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-05-07 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-05-08 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-05-09 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-05-10 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-05-11 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-05-12 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-05-13 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-05-14 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-05-15 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-05-16 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-05-17 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-05-18 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-05-19 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-05-20 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-05-21 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-05-22 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-05-23 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-05-24 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-05-25 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-05-26 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-05-27 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-05-28 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-05-29 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-05-30 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-05-31 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-06-01 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-06-02 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-06-03 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-06-04 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-06-05 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-06-06 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-06-07 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-06-08 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-06-09 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-06-10 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-06-11 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-06-12 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-06-13 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-06-14 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-06-15 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-06-16 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-06-17 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-06-18 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-06-19 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-06-20 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-06-21 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-06-22 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-06-23 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-06-24 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-06-25 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-06-26 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-06-27 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-06-28 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-06-29 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-06-30 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-07-01 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-07-02 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-07-03 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-07-04 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-07-05 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-07-06 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-07-07 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-07-08 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-07-09 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-07-10 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-07-11 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-07-12 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-07-13 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-07-14 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-07-15 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-07-16 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-07-17 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-07-18 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-07-19 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-07-20 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-07-21 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-07-22 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-07-23 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-07-24 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-07-25 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-07-26 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-07-27 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-07-28 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-07-29 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-07-30 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-07-31 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-08-01 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-08-02 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-08-03 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-08-04 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-08-05 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-08-06 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-08-07 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-08-08 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-08-09 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-08-10 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-08-11 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-08-12 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-08-13 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-08-14 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-08-15 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-08-16 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-08-17 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-08-18 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-08-19 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-08-20 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-08-21 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-08-22 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-08-23 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-08-24 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-08-25 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-08-26 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-08-27 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-08-28 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-08-29 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-08-30 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-08-31 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-09-01 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-09-02 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-09-03 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-09-04 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-09-05 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-09-06 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-09-07 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-09-08 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-09-09 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-09-10 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-09-11 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-09-12 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-09-13 [lineitem_partitioned] /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-09-14 [lineitem_partitioned] ################################################################### Below is when masking is disabled ################################################################### Explain STAGE DEPENDENCIES: Stage-1 is a root stage Stage-2 depends on stages: Stage-1 Stage-0 depends on stages: Stage-2 Stage-3 depends on stages: Stage-0 STAGE PLANS: Stage: Stage-1 Tez DagId: hive_20190310233407_f6c7a9fd-ac40-47f9-bd3a-66123e6f2b34:15 Edges: Reducer 2 <- Map 1 (SIMPLE_EDGE) DagName: Vertices: Map 1 Map Operator Tree: TableScan alias: lineitem_partitioned Statistics: Num rows: 576 Data size: 115222 Basic stats: COMPLETE Column stats: NONE GatherStats: false Select Operator expressions: l_commitdate (type: string), l_receiptdate (type: string) outputColumnNames: l_commitdate, l_receiptdate Statistics: Num rows: 576 Data size: 115222 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: l_commitdate (type: string), l_receiptdate (type: string) mode: hash outputColumnNames: _col0, _col1 Statistics: Num rows: 576 Data size: 115222 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) null sort order: aa sort order: ++ Map-reduce partition columns: _col0 (type: string), _col1 (type: string) Statistics: Num rows: 576 Data size: 115222 Basic stats: COMPLETE Column stats: NONE tag: -1 auto parallelism: true Path -> Alias: /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-01-02 [lineitem_partitioned] Path -> Partition: /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-01-02 Partition base file name: l_shipdate=1992-01-02 input format: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat output format: org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat partition values: l_shipdate 1992-01-02 properties: bucket_count -1 column.name.delimiter , columns l_orderkey,l_partkey,l_suppkey,l_linenumber,l_quantity,l_extendedprice,l_discount,l_tax,l_returnflag,l_linestatus,l_commitdate,l_receiptdate,l_shipinstruct,l_shipmode,l_comment columns.comments columns.types int:int:int:int:double:double:double:double:string:string:string:string:string:string:string file.inputformat org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat file.outputformat org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat location /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-01-02 name default.lineitem_partitioned numFiles 1 partition_columns l_shipdate partition_columns.types string serialization.ddl struct lineitem_partitioned { i32 l_orderkey, i32 l_partkey, i32 l_suppkey, i32 l_linenumber, double l_quantity, double l_extendedprice, double l_discount, double l_tax, string l_returnflag, string l_linestatus, string l_commitdate, string l_receiptdate, string l_shipinstruct, string l_shipmode, string l_comment} serialization.format 1 serialization.lib org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe totalSize 115222 transient_lastDdlTime 1552248455 serde: org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe input format: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat output format: org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat properties: EXTERNAL TRUE bucket_count -1 column.name.delimiter , columns l_orderkey,l_partkey,l_suppkey,l_linenumber,l_quantity,l_extendedprice,l_discount,l_tax,l_returnflag,l_linestatus,l_commitdate,l_receiptdate,l_shipinstruct,l_shipmode,l_comment columns.comments columns.types int:int:int:int:double:double:double:double:string:string:string:string:string:string:string file.inputformat org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat file.outputformat org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat location /user/analyst1//tpch/sf100/lineitem_partitioned name default.lineitem_partitioned partition_columns l_shipdate partition_columns.types string serialization.ddl struct lineitem_partitioned { i32 l_orderkey, i32 l_partkey, i32 l_suppkey, i32 l_linenumber, double l_quantity, double l_extendedprice, double l_discount, double l_tax, string l_returnflag, string l_linestatus, string l_commitdate, string l_receiptdate, string l_shipinstruct, string l_shipmode, string l_comment} serialization.format 1 serialization.lib org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe transient_lastDdlTime 1552248302 serde: org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe name: default.lineitem_partitioned name: default.lineitem_partitioned Truncated Path -> Alias: /user/analyst1//tpch/sf100/lineitem_partitioned/l_shipdate=1992-01-02 [lineitem_partitioned] Reducer 2 Needs Tagging: false Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: string), KEY._col1 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 Statistics: Num rows: 288 Data size: 57611 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), '1992-01-02' (type: string) outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 288 Data size: 57611 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 1 directory: hdfs://ip-172-31-11-52.ec2.internal:8020/user/analyst1/tpch/sf100/lineitem_partitioned/.hive-staging_hive_2019-03-10_23-34-07_300_7580714612909031439-7/-ext-10000 NumFilesPerFileSink: 1 Statistics: Num rows: 288 Data size: 57611 Basic stats: COMPLETE Column stats: NONE Stats Publishing Key Prefix: hdfs://ip-172-31-11-52.ec2.internal:8020/user/analyst1/tpch/sf100/lineitem_partitioned/.hive-staging_hive_2019-03-10_23-34-07_300_7580714612909031439-7/-ext-10000/ table: input format: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat output format: org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat properties: EXTERNAL TRUE bucket_count -1 column.name.delimiter , columns l_commitdate,l_receiptdate columns.comments columns.types string:string file.inputformat org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat file.outputformat org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat location hdfs://ip-172-31-11-52.ec2.internal:8020/user/analyst1/tpch/sf100/lineitem_partitioned name analyst1.test1 partition_columns l_shipdate partition_columns.types string serialization.ddl struct test1 { string l_commitdate, string l_receiptdate} serialization.format 1 serialization.lib org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe transient_lastDdlTime 1552260295 serde: org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe name: analyst1.test1 TotalFiles: 1 GatherStats: true MultiFileSpray: false Stage: Stage-2 Dependency Collection Stage: Stage-0 Move Operator tables: partition: l_shipdate replace: true source: hdfs://ip-172-31-11-52.ec2.internal:8020/user/analyst1/tpch/sf100/lineitem_partitioned/.hive-staging_hive_2019-03-10_23-34-07_300_7580714612909031439-7/-ext-10000 table: input format: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat output format: org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat properties: EXTERNAL TRUE bucket_count -1 column.name.delimiter , columns l_commitdate,l_receiptdate columns.comments columns.types string:string file.inputformat org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat file.outputformat org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat location hdfs://ip-172-31-11-52.ec2.internal:8020/user/analyst1/tpch/sf100/lineitem_partitioned name analyst1.test1 partition_columns l_shipdate partition_columns.types string serialization.ddl struct test1 { string l_commitdate, string l_receiptdate} serialization.format 1 serialization.lib org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe transient_lastDdlTime 1552260295 serde: org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe name: analyst1.test1 Stage: Stage-3 Stats-Aggr Operator Stats Aggregation Key Prefix: hdfs://ip-172-31-11-52.ec2.internal:8020/user/analyst1/tpch/sf100/lineitem_partitioned/.hive-staging_hive_2019-03-10_23-34-07_300_7580714612909031439-7/-ext-10000/