diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/DynamicPartitionPruningOptimization.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/DynamicPartitionPruningOptimization.java index c8691e8..8692c45 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/DynamicPartitionPruningOptimization.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/DynamicPartitionPruningOptimization.java @@ -398,25 +398,33 @@ private boolean generateSemiJoinOperatorPlan(DynamicListContext ctx, ParseContex String internalColName = null; ExprNodeDesc exprNodeDesc = key; // Find the ExprNodeColumnDesc - while (!(exprNodeDesc instanceof ExprNodeColumnDesc)) { + while (!(exprNodeDesc instanceof ExprNodeColumnDesc) && + (exprNodeDesc.getChildren() != null)) { exprNodeDesc = exprNodeDesc.getChildren().get(0); } - internalColName = ((ExprNodeColumnDesc) exprNodeDesc).getColumn(); - ExprNodeColumnDesc colExpr = ((ExprNodeColumnDesc)(parentOfRS. - getColumnExprMap().get(internalColName))); - String colName = ExprNodeDescUtils.extractColName(colExpr); + if (exprNodeDesc instanceof ExprNodeColumnDesc) { + internalColName = ((ExprNodeColumnDesc) exprNodeDesc).getColumn(); - // Fetch the TableScan Operator. - Operator op = parentOfRS.getParentOperators().get(0); - while (op != null && !(op instanceof TableScanOperator)) { - op = op.getParentOperators().get(0); - } - assert op != null; + ExprNodeColumnDesc colExpr = ((ExprNodeColumnDesc) (parentOfRS. + getColumnExprMap().get(internalColName))); + String colName = ExprNodeDescUtils.extractColName(colExpr); + + // Fetch the TableScan Operator. + Operator op = parentOfRS.getParentOperators().get(0); + while (op != null && !(op instanceof TableScanOperator)) { + op = op.getParentOperators().get(0); + } + assert op != null; - Table table = ((TableScanOperator) op).getConf().getTableMetadata(); - if (table.isPartitionKey(colName)) { - // The column is partition column, skip the optimization. + Table table = ((TableScanOperator) op).getConf().getTableMetadata(); + if (table.isPartitionKey(colName)) { + // The column is partition column, skip the optimization. + return false; + } + } else { + // No column found! + // Bail out return false; } } diff --git a/ql/src/test/queries/clientpositive/dynamic_semijoin_reduction_2.q b/ql/src/test/queries/clientpositive/dynamic_semijoin_reduction_2.q new file mode 100644 index 0000000..fb1dfb9 --- /dev/null +++ b/ql/src/test/queries/clientpositive/dynamic_semijoin_reduction_2.q @@ -0,0 +1,155 @@ +set hive.compute.query.using.stats=false; +set hive.mapred.mode=nonstrict; +set hive.explain.user=false; +set hive.optimize.ppd=true; +set hive.ppd.remove.duplicatefilters=true; +set hive.tez.dynamic.partition.pruning=true; +set hive.tez.dynamic.semijoin.reduction=true; +set hive.optimize.metadataonly=false; +set hive.optimize.index.filter=true; + +CREATE TABLE `table_1`( + `timestamp_col_1` timestamp, + `decimal3003_col_2` decimal(30,3), + `tinyint_col_3` tinyint, + `decimal0101_col_4` decimal(1,1), + `boolean_col_5` boolean, + `float_col_6` float, + `bigint_col_7` bigint, + `varchar0098_col_8` varchar(98), + `timestamp_col_9` timestamp, + `bigint_col_10` bigint, + `decimal0903_col_11` decimal(9,3), + `timestamp_col_12` timestamp, + `timestamp_col_13` timestamp, + `float_col_14` float, + `char0254_col_15` char(254), + `double_col_16` double, + `timestamp_col_17` timestamp, + `boolean_col_18` boolean, + `decimal2608_col_19` decimal(26,8), + `varchar0216_col_20` varchar(216), + `string_col_21` string, + `bigint_col_22` bigint, + `boolean_col_23` boolean, + `timestamp_col_24` timestamp, + `boolean_col_25` boolean, + `decimal2016_col_26` decimal(20,16), + `string_col_27` string, + `decimal0202_col_28` decimal(2,2), + `float_col_29` float, + `decimal2020_col_30` decimal(20,20), + `boolean_col_31` boolean, + `double_col_32` double, + `varchar0148_col_33` varchar(148), + `decimal2121_col_34` decimal(21,21), + `tinyint_col_35` tinyint, + `boolean_col_36` boolean, + `boolean_col_37` boolean, + `string_col_38` string, + `decimal3420_col_39` decimal(34,20), + `timestamp_col_40` timestamp, + `decimal1408_col_41` decimal(14,8), + `string_col_42` string, + `decimal0902_col_43` decimal(9,2), + `varchar0204_col_44` varchar(204), + `boolean_col_45` boolean, + `timestamp_col_46` timestamp, + `boolean_col_47` boolean, + `bigint_col_48` bigint, + `boolean_col_49` boolean, + `smallint_col_50` smallint, + `decimal0704_col_51` decimal(7,4), + `timestamp_col_52` timestamp, + `boolean_col_53` boolean, + `timestamp_col_54` timestamp, + `int_col_55` int, + `decimal0505_col_56` decimal(5,5), + `char0155_col_57` char(155), + `boolean_col_58` boolean, + `bigint_col_59` bigint, + `boolean_col_60` boolean, + `boolean_col_61` boolean, + `char0249_col_62` char(249), + `boolean_col_63` boolean, + `timestamp_col_64` timestamp, + `decimal1309_col_65` decimal(13,9), + `int_col_66` int, + `float_col_67` float, + `timestamp_col_68` timestamp, + `timestamp_col_69` timestamp, + `boolean_col_70` boolean, + `timestamp_col_71` timestamp, + `double_col_72` double, + `boolean_col_73` boolean, + `char0222_col_74` char(222), + `float_col_75` float, + `string_col_76` string, + `decimal2612_col_77` decimal(26,12), + `timestamp_col_78` timestamp, + `char0128_col_79` char(128), + `timestamp_col_80` timestamp, + `double_col_81` double, + `timestamp_col_82` timestamp, + `float_col_83` float, + `decimal2622_col_84` decimal(26,22), + `double_col_85` double, + `float_col_86` float, + `decimal0907_col_87` decimal(9,7)) +ROW FORMAT SERDE + 'org.apache.hadoop.hive.ql.io.orc.OrcSerde' +STORED AS INPUTFORMAT + 'org.apache.hadoop.hive.ql.io.orc.OrcInputFormat' +OUTPUTFORMAT + 'org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat'; + +CREATE TABLE `table_18`( + `boolean_col_1` boolean, + `boolean_col_2` boolean, + `decimal2518_col_3` decimal(25,18), + `float_col_4` float, + `timestamp_col_5` timestamp, + `double_col_6` double, + `double_col_7` double, + `char0035_col_8` char(35), + `decimal2709_col_9` decimal(27,9), + `int_col_10` int, + `timestamp_col_11` timestamp, + `decimal3604_col_12` decimal(36,4), + `string_col_13` string, + `int_col_14` int, + `tinyint_col_15` tinyint, + `decimal1911_col_16` decimal(19,11), + `float_col_17` float, + `timestamp_col_18` timestamp, + `smallint_col_19` smallint, + `tinyint_col_20` tinyint, + `timestamp_col_21` timestamp, + `boolean_col_22` boolean, + `int_col_23` int) +ROW FORMAT SERDE + 'org.apache.hadoop.hive.ql.io.orc.OrcSerde' +STORED AS INPUTFORMAT + 'org.apache.hadoop.hive.ql.io.orc.OrcInputFormat' +OUTPUTFORMAT + 'org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat'; + +-- HIVE-15904 +EXPLAIN +SELECT +COALESCE(498, LEAD(COALESCE(-973, -684, 515)) OVER (PARTITION BY (t2.int_col_10 + t1.smallint_col_50) ORDER BY (t2.int_col_10 + t1.smallint_col_50), FLOOR(t1.double_col_16) DESC), 524) AS int_col, +(t2.int_col_10) + (t1.smallint_col_50) AS int_col_1, +FLOOR(t1.double_col_16) AS float_col, +COALESCE(SUM(COALESCE(62, -380, -435)) OVER (PARTITION BY (t2.int_col_10 + t1.smallint_col_50) ORDER BY (t2.int_col_10 + t1.smallint_col_50) DESC, FLOOR(t1.double_col_16) DESC ROWS BETWEEN UNBOUNDED PRECEDING AND 48 FOLLOWING), 704) AS int_col_2 +FROM table_1 t1 +INNER JOIN table_18 t2 ON (((t2.tinyint_col_15) = (t1.bigint_col_7)) AND +((t2.decimal2709_col_9) = (t1.decimal2016_col_26))) AND +((t2.tinyint_col_20) = (t1.tinyint_col_3)) +WHERE (t2.smallint_col_19) IN (SELECT +COALESCE(-92, -994) AS int_col +FROM table_1 tt1 +INNER JOIN table_18 tt2 ON (tt2.decimal1911_col_16) = (tt1.decimal2612_col_77) +WHERE (t1.timestamp_col_9) = (tt2.timestamp_col_18)); + +drop table table_1; +drop table table_18; diff --git a/ql/src/test/results/clientpositive/llap/dynamic_semijoin_reduction_2.q.out b/ql/src/test/results/clientpositive/llap/dynamic_semijoin_reduction_2.q.out new file mode 100644 index 0000000..9d4e5e5 --- /dev/null +++ b/ql/src/test/results/clientpositive/llap/dynamic_semijoin_reduction_2.q.out @@ -0,0 +1,590 @@ +PREHOOK: query: CREATE TABLE `table_1`( + `timestamp_col_1` timestamp, + `decimal3003_col_2` decimal(30,3), + `tinyint_col_3` tinyint, + `decimal0101_col_4` decimal(1,1), + `boolean_col_5` boolean, + `float_col_6` float, + `bigint_col_7` bigint, + `varchar0098_col_8` varchar(98), + `timestamp_col_9` timestamp, + `bigint_col_10` bigint, + `decimal0903_col_11` decimal(9,3), + `timestamp_col_12` timestamp, + `timestamp_col_13` timestamp, + `float_col_14` float, + `char0254_col_15` char(254), + `double_col_16` double, + `timestamp_col_17` timestamp, + `boolean_col_18` boolean, + `decimal2608_col_19` decimal(26,8), + `varchar0216_col_20` varchar(216), + `string_col_21` string, + `bigint_col_22` bigint, + `boolean_col_23` boolean, + `timestamp_col_24` timestamp, + `boolean_col_25` boolean, + `decimal2016_col_26` decimal(20,16), + `string_col_27` string, + `decimal0202_col_28` decimal(2,2), + `float_col_29` float, + `decimal2020_col_30` decimal(20,20), + `boolean_col_31` boolean, + `double_col_32` double, + `varchar0148_col_33` varchar(148), + `decimal2121_col_34` decimal(21,21), + `tinyint_col_35` tinyint, + `boolean_col_36` boolean, + `boolean_col_37` boolean, + `string_col_38` string, + `decimal3420_col_39` decimal(34,20), + `timestamp_col_40` timestamp, + `decimal1408_col_41` decimal(14,8), + `string_col_42` string, + `decimal0902_col_43` decimal(9,2), + `varchar0204_col_44` varchar(204), + `boolean_col_45` boolean, + `timestamp_col_46` timestamp, + `boolean_col_47` boolean, + `bigint_col_48` bigint, + `boolean_col_49` boolean, + `smallint_col_50` smallint, + `decimal0704_col_51` decimal(7,4), + `timestamp_col_52` timestamp, + `boolean_col_53` boolean, + `timestamp_col_54` timestamp, + `int_col_55` int, + `decimal0505_col_56` decimal(5,5), + `char0155_col_57` char(155), + `boolean_col_58` boolean, + `bigint_col_59` bigint, + `boolean_col_60` boolean, + `boolean_col_61` boolean, + `char0249_col_62` char(249), + `boolean_col_63` boolean, + `timestamp_col_64` timestamp, + `decimal1309_col_65` decimal(13,9), + `int_col_66` int, + `float_col_67` float, + `timestamp_col_68` timestamp, + `timestamp_col_69` timestamp, + `boolean_col_70` boolean, + `timestamp_col_71` timestamp, + `double_col_72` double, + `boolean_col_73` boolean, + `char0222_col_74` char(222), + `float_col_75` float, + `string_col_76` string, + `decimal2612_col_77` decimal(26,12), + `timestamp_col_78` timestamp, + `char0128_col_79` char(128), + `timestamp_col_80` timestamp, + `double_col_81` double, + `timestamp_col_82` timestamp, + `float_col_83` float, + `decimal2622_col_84` decimal(26,22), + `double_col_85` double, + `float_col_86` float, + `decimal0907_col_87` decimal(9,7)) +ROW FORMAT SERDE + 'org.apache.hadoop.hive.ql.io.orc.OrcSerde' +STORED AS INPUTFORMAT + 'org.apache.hadoop.hive.ql.io.orc.OrcInputFormat' +OUTPUTFORMAT + 'org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat' +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@table_1 +POSTHOOK: query: CREATE TABLE `table_1`( + `timestamp_col_1` timestamp, + `decimal3003_col_2` decimal(30,3), + `tinyint_col_3` tinyint, + `decimal0101_col_4` decimal(1,1), + `boolean_col_5` boolean, + `float_col_6` float, + `bigint_col_7` bigint, + `varchar0098_col_8` varchar(98), + `timestamp_col_9` timestamp, + `bigint_col_10` bigint, + `decimal0903_col_11` decimal(9,3), + `timestamp_col_12` timestamp, + `timestamp_col_13` timestamp, + `float_col_14` float, + `char0254_col_15` char(254), + `double_col_16` double, + `timestamp_col_17` timestamp, + `boolean_col_18` boolean, + `decimal2608_col_19` decimal(26,8), + `varchar0216_col_20` varchar(216), + `string_col_21` string, + `bigint_col_22` bigint, + `boolean_col_23` boolean, + `timestamp_col_24` timestamp, + `boolean_col_25` boolean, + `decimal2016_col_26` decimal(20,16), + `string_col_27` string, + `decimal0202_col_28` decimal(2,2), + `float_col_29` float, + `decimal2020_col_30` decimal(20,20), + `boolean_col_31` boolean, + `double_col_32` double, + `varchar0148_col_33` varchar(148), + `decimal2121_col_34` decimal(21,21), + `tinyint_col_35` tinyint, + `boolean_col_36` boolean, + `boolean_col_37` boolean, + `string_col_38` string, + `decimal3420_col_39` decimal(34,20), + `timestamp_col_40` timestamp, + `decimal1408_col_41` decimal(14,8), + `string_col_42` string, + `decimal0902_col_43` decimal(9,2), + `varchar0204_col_44` varchar(204), + `boolean_col_45` boolean, + `timestamp_col_46` timestamp, + `boolean_col_47` boolean, + `bigint_col_48` bigint, + `boolean_col_49` boolean, + `smallint_col_50` smallint, + `decimal0704_col_51` decimal(7,4), + `timestamp_col_52` timestamp, + `boolean_col_53` boolean, + `timestamp_col_54` timestamp, + `int_col_55` int, + `decimal0505_col_56` decimal(5,5), + `char0155_col_57` char(155), + `boolean_col_58` boolean, + `bigint_col_59` bigint, + `boolean_col_60` boolean, + `boolean_col_61` boolean, + `char0249_col_62` char(249), + `boolean_col_63` boolean, + `timestamp_col_64` timestamp, + `decimal1309_col_65` decimal(13,9), + `int_col_66` int, + `float_col_67` float, + `timestamp_col_68` timestamp, + `timestamp_col_69` timestamp, + `boolean_col_70` boolean, + `timestamp_col_71` timestamp, + `double_col_72` double, + `boolean_col_73` boolean, + `char0222_col_74` char(222), + `float_col_75` float, + `string_col_76` string, + `decimal2612_col_77` decimal(26,12), + `timestamp_col_78` timestamp, + `char0128_col_79` char(128), + `timestamp_col_80` timestamp, + `double_col_81` double, + `timestamp_col_82` timestamp, + `float_col_83` float, + `decimal2622_col_84` decimal(26,22), + `double_col_85` double, + `float_col_86` float, + `decimal0907_col_87` decimal(9,7)) +ROW FORMAT SERDE + 'org.apache.hadoop.hive.ql.io.orc.OrcSerde' +STORED AS INPUTFORMAT + 'org.apache.hadoop.hive.ql.io.orc.OrcInputFormat' +OUTPUTFORMAT + 'org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat' +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@table_1 +PREHOOK: query: CREATE TABLE `table_18`( + `boolean_col_1` boolean, + `boolean_col_2` boolean, + `decimal2518_col_3` decimal(25,18), + `float_col_4` float, + `timestamp_col_5` timestamp, + `double_col_6` double, + `double_col_7` double, + `char0035_col_8` char(35), + `decimal2709_col_9` decimal(27,9), + `int_col_10` int, + `timestamp_col_11` timestamp, + `decimal3604_col_12` decimal(36,4), + `string_col_13` string, + `int_col_14` int, + `tinyint_col_15` tinyint, + `decimal1911_col_16` decimal(19,11), + `float_col_17` float, + `timestamp_col_18` timestamp, + `smallint_col_19` smallint, + `tinyint_col_20` tinyint, + `timestamp_col_21` timestamp, + `boolean_col_22` boolean, + `int_col_23` int) +ROW FORMAT SERDE + 'org.apache.hadoop.hive.ql.io.orc.OrcSerde' +STORED AS INPUTFORMAT + 'org.apache.hadoop.hive.ql.io.orc.OrcInputFormat' +OUTPUTFORMAT + 'org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat' +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@table_18 +POSTHOOK: query: CREATE TABLE `table_18`( + `boolean_col_1` boolean, + `boolean_col_2` boolean, + `decimal2518_col_3` decimal(25,18), + `float_col_4` float, + `timestamp_col_5` timestamp, + `double_col_6` double, + `double_col_7` double, + `char0035_col_8` char(35), + `decimal2709_col_9` decimal(27,9), + `int_col_10` int, + `timestamp_col_11` timestamp, + `decimal3604_col_12` decimal(36,4), + `string_col_13` string, + `int_col_14` int, + `tinyint_col_15` tinyint, + `decimal1911_col_16` decimal(19,11), + `float_col_17` float, + `timestamp_col_18` timestamp, + `smallint_col_19` smallint, + `tinyint_col_20` tinyint, + `timestamp_col_21` timestamp, + `boolean_col_22` boolean, + `int_col_23` int) +ROW FORMAT SERDE + 'org.apache.hadoop.hive.ql.io.orc.OrcSerde' +STORED AS INPUTFORMAT + 'org.apache.hadoop.hive.ql.io.orc.OrcInputFormat' +OUTPUTFORMAT + 'org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat' +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@table_18 +PREHOOK: query: EXPLAIN +SELECT +COALESCE(498, LEAD(COALESCE(-973, -684, 515)) OVER (PARTITION BY (t2.int_col_10 + t1.smallint_col_50) ORDER BY (t2.int_col_10 + t1.smallint_col_50), FLOOR(t1.double_col_16) DESC), 524) AS int_col, +(t2.int_col_10) + (t1.smallint_col_50) AS int_col_1, +FLOOR(t1.double_col_16) AS float_col, +COALESCE(SUM(COALESCE(62, -380, -435)) OVER (PARTITION BY (t2.int_col_10 + t1.smallint_col_50) ORDER BY (t2.int_col_10 + t1.smallint_col_50) DESC, FLOOR(t1.double_col_16) DESC ROWS BETWEEN UNBOUNDED PRECEDING AND 48 FOLLOWING), 704) AS int_col_2 +FROM table_1 t1 +INNER JOIN table_18 t2 ON (((t2.tinyint_col_15) = (t1.bigint_col_7)) AND +((t2.decimal2709_col_9) = (t1.decimal2016_col_26))) AND +((t2.tinyint_col_20) = (t1.tinyint_col_3)) +WHERE (t2.smallint_col_19) IN (SELECT +COALESCE(-92, -994) AS int_col +FROM table_1 tt1 +INNER JOIN table_18 tt2 ON (tt2.decimal1911_col_16) = (tt1.decimal2612_col_77) +WHERE (t1.timestamp_col_9) = (tt2.timestamp_col_18)) +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN +SELECT +COALESCE(498, LEAD(COALESCE(-973, -684, 515)) OVER (PARTITION BY (t2.int_col_10 + t1.smallint_col_50) ORDER BY (t2.int_col_10 + t1.smallint_col_50), FLOOR(t1.double_col_16) DESC), 524) AS int_col, +(t2.int_col_10) + (t1.smallint_col_50) AS int_col_1, +FLOOR(t1.double_col_16) AS float_col, +COALESCE(SUM(COALESCE(62, -380, -435)) OVER (PARTITION BY (t2.int_col_10 + t1.smallint_col_50) ORDER BY (t2.int_col_10 + t1.smallint_col_50) DESC, FLOOR(t1.double_col_16) DESC ROWS BETWEEN UNBOUNDED PRECEDING AND 48 FOLLOWING), 704) AS int_col_2 +FROM table_1 t1 +INNER JOIN table_18 t2 ON (((t2.tinyint_col_15) = (t1.bigint_col_7)) AND +((t2.decimal2709_col_9) = (t1.decimal2016_col_26))) AND +((t2.tinyint_col_20) = (t1.tinyint_col_3)) +WHERE (t2.smallint_col_19) IN (SELECT +COALESCE(-92, -994) AS int_col +FROM table_1 tt1 +INNER JOIN table_18 tt2 ON (tt2.decimal1911_col_16) = (tt1.decimal2612_col_77) +WHERE (t1.timestamp_col_9) = (tt2.timestamp_col_18)) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Map 11 <- Reducer 6 (BROADCAST_EDGE) + Reducer 10 <- Reducer 9 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) + Reducer 3 <- Reducer 10 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) + Reducer 4 <- Reducer 3 (SIMPLE_EDGE) + Reducer 5 <- Reducer 4 (SIMPLE_EDGE) + Reducer 6 <- Reducer 2 (CUSTOM_SIMPLE_EDGE) + Reducer 9 <- Map 11 (SIMPLE_EDGE), Map 8 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: t1 + filterExpr: (tinyint_col_3 is not null and bigint_col_7 is not null and decimal2016_col_26 is not null) (type: boolean) + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Filter Operator + predicate: (tinyint_col_3 is not null and bigint_col_7 is not null and decimal2016_col_26 is not null) (type: boolean) + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Select Operator + expressions: tinyint_col_3 (type: tinyint), bigint_col_7 (type: bigint), timestamp_col_9 (type: timestamp), double_col_16 (type: double), decimal2016_col_26 (type: decimal(20,16)), smallint_col_50 (type: smallint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: tinyint), _col4 (type: decimal(34,16)), _col1 (type: bigint) + sort order: +++ + Map-reduce partition columns: _col0 (type: tinyint), _col4 (type: decimal(34,16)), _col1 (type: bigint) + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + value expressions: _col2 (type: timestamp), _col3 (type: double), _col5 (type: smallint) + Execution mode: llap + LLAP IO: all inputs + Map 11 + Map Operator Tree: + TableScan + alias: tt2 + filterExpr: ((timestamp_col_18 = timestamp_col_18) and decimal1911_col_16 is not null and timestamp_col_18 BETWEEN DynamicValue(RS_23_t1_timestamp_col_18_min) AND DynamicValue(RS_23_t1_timestamp_col_18_max) and in_bloom_filter(timestamp_col_18, DynamicValue(RS_23_t1_timestamp_col_18_bloom_filter))) (type: boolean) + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Filter Operator + predicate: ((timestamp_col_18 = timestamp_col_18) and decimal1911_col_16 is not null and timestamp_col_18 BETWEEN DynamicValue(RS_23_t1_timestamp_col_18_min) AND DynamicValue(RS_23_t1_timestamp_col_18_max) and in_bloom_filter(timestamp_col_18, DynamicValue(RS_23_t1_timestamp_col_18_bloom_filter))) (type: boolean) + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Select Operator + expressions: decimal1911_col_16 (type: decimal(19,11)), timestamp_col_18 (type: timestamp) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: decimal(26,12)) + sort order: + + Map-reduce partition columns: _col0 (type: decimal(26,12)) + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + value expressions: _col1 (type: timestamp) + Execution mode: llap + LLAP IO: all inputs + Map 7 + Map Operator Tree: + TableScan + alias: t2 + filterExpr: (tinyint_col_20 is not null and decimal2709_col_9 is not null and tinyint_col_15 is not null) (type: boolean) + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Filter Operator + predicate: (tinyint_col_20 is not null and decimal2709_col_9 is not null and tinyint_col_15 is not null) (type: boolean) + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Select Operator + expressions: decimal2709_col_9 (type: decimal(27,9)), int_col_10 (type: int), tinyint_col_15 (type: tinyint), smallint_col_19 (type: smallint), tinyint_col_20 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Reduce Output Operator + key expressions: _col4 (type: tinyint), _col0 (type: decimal(34,16)), UDFToLong(_col2) (type: bigint) + sort order: +++ + Map-reduce partition columns: _col4 (type: tinyint), _col0 (type: decimal(34,16)), UDFToLong(_col2) (type: bigint) + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + value expressions: _col1 (type: int), _col3 (type: smallint) + Execution mode: llap + LLAP IO: all inputs + Map 8 + Map Operator Tree: + TableScan + alias: tt1 + filterExpr: decimal2612_col_77 is not null (type: boolean) + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Filter Operator + predicate: decimal2612_col_77 is not null (type: boolean) + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Select Operator + expressions: decimal2612_col_77 (type: decimal(26,12)) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: decimal(26,12)) + sort order: + + Map-reduce partition columns: _col0 (type: decimal(26,12)) + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Execution mode: llap + LLAP IO: all inputs + Reducer 10 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + keys: KEY._col0 (type: timestamp) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Select Operator + expressions: _col0 (type: timestamp) + outputColumnNames: _col1 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: timestamp), -92 (type: int) + sort order: ++ + Map-reduce partition columns: _col1 (type: timestamp), -92 (type: int) + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: tinyint), _col4 (type: decimal(34,16)), _col1 (type: bigint) + 1 _col4 (type: tinyint), _col0 (type: decimal(34,16)), UDFToLong(_col2) (type: bigint) + outputColumnNames: _col2, _col3, _col5, _col7, _col9 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Reduce Output Operator + key expressions: _col2 (type: timestamp), UDFToInteger(_col9) (type: int) + sort order: ++ + Map-reduce partition columns: _col2 (type: timestamp), UDFToInteger(_col9) (type: int) + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + value expressions: _col3 (type: double), _col5 (type: smallint), _col7 (type: int) + Select Operator + expressions: _col2 (type: timestamp) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Group By Operator + aggregations: min(_col0), max(_col0), bloom_filter(_col0, expectedEntries=1) + mode: hash + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 120 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 120 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: timestamp), _col1 (type: timestamp), _col2 (type: binary) + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col2 (type: timestamp), UDFToInteger(_col9) (type: int) + 1 _col1 (type: timestamp), -92 (type: int) + outputColumnNames: _col3, _col5, _col7 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Reduce Output Operator + key expressions: (_col7 + UDFToInteger(_col5)) (type: int), floor(_col3) (type: bigint) + sort order: +- + Map-reduce partition columns: (_col7 + UDFToInteger(_col5)) (type: int) + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + value expressions: _col3 (type: double), _col5 (type: smallint), _col7 (type: int) + Reducer 4 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: VALUE._col3 (type: double), VALUE._col5 (type: smallint), VALUE._col7 (type: int) + outputColumnNames: _col3, _col5, _col7 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col3: double, _col5: smallint, _col7: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: (_col7 + UDFToInteger(_col5)) ASC NULLS FIRST, floor(_col3) DESC NULLS LAST + partition by: (_col7 + UDFToInteger(_col5)) + raw input shape: + window functions: + window function definition + alias: LEAD_window_0 + arguments: -973 + name: LEAD + window function: GenericUDAFLeadEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Select Operator + expressions: LEAD_window_0 (type: int), _col3 (type: double), _col5 (type: smallint), _col7 (type: int) + outputColumnNames: LEAD_window_0, _col3, _col5, _col7 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Reduce Output Operator + key expressions: (_col7 + UDFToInteger(_col5)) (type: int), floor(_col3) (type: bigint) + sort order: -- + Map-reduce partition columns: (_col7 + UDFToInteger(_col5)) (type: int) + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + value expressions: LEAD_window_0 (type: int), _col3 (type: double), _col5 (type: smallint), _col7 (type: int) + Reducer 5 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: int), VALUE._col4 (type: double), VALUE._col6 (type: smallint), VALUE._col8 (type: int) + outputColumnNames: _col0, _col4, _col6, _col8 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: int, _col4: double, _col6: smallint, _col8: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: (_col8 + UDFToInteger(_col6)) DESC NULLS LAST, floor(_col4) DESC NULLS LAST + partition by: (_col8 + UDFToInteger(_col6)) + raw input shape: + window functions: + window function definition + alias: sum_window_1 + arguments: 62 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~FOLLOWING(48) + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Select Operator + expressions: COALESCE(498,_col0,524) (type: int), (_col8 + UDFToInteger(_col6)) (type: int), floor(_col4) (type: bigint), COALESCE(sum_window_1,704) (type: bigint) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Reducer 6 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: min(VALUE._col0), max(VALUE._col1), bloom_filter(VALUE._col2, expectedEntries=1) + mode: final + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 120 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 120 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: timestamp), _col1 (type: timestamp), _col2 (type: binary) + Reducer 9 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: decimal(26,12)) + 1 _col0 (type: decimal(26,12)) + outputColumnNames: _col2 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Group By Operator + keys: _col2 (type: timestamp) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: timestamp) + sort order: + + Map-reduce partition columns: _col0 (type: timestamp) + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: drop table table_1 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@table_1 +PREHOOK: Output: default@table_1 +POSTHOOK: query: drop table table_1 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@table_1 +POSTHOOK: Output: default@table_1 +PREHOOK: query: drop table table_18 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@table_18 +PREHOOK: Output: default@table_18 +POSTHOOK: query: drop table table_18 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@table_18 +POSTHOOK: Output: default@table_18