diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java index f50f4d3..1c44ade 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java @@ -77,14 +77,12 @@ import org.apache.calcite.rel.type.RelDataTypeFactory; import org.apache.calcite.rel.type.RelDataTypeField; import org.apache.calcite.rex.RexBuilder; -import org.apache.calcite.rex.RexExecutorImpl; import org.apache.calcite.rex.RexFieldCollation; import org.apache.calcite.rex.RexInputRef; import org.apache.calcite.rex.RexNode; import org.apache.calcite.rex.RexUtil; import org.apache.calcite.rex.RexWindowBound; import org.apache.calcite.schema.SchemaPlus; -import org.apache.calcite.schema.Schemas; import org.apache.calcite.sql.SqlAggFunction; import org.apache.calcite.sql.SqlCall; import org.apache.calcite.sql.SqlExplainLevel; @@ -1367,24 +1365,11 @@ private RelNode genUnionLogicalPlan(String unionalias, String leftalias, RelNode private RelNode genJoinRelNode(RelNode leftRel, RelNode rightRel, JoinType hiveJoinType, ASTNode joinCond) throws SemanticException { - RelNode joinRel = null; - // 1. construct the RowResolver for the new Join Node by combining row - // resolvers from left, right RowResolver leftRR = this.relToHiveRR.get(leftRel); RowResolver rightRR = this.relToHiveRR.get(rightRel); - RowResolver joinRR = null; - if (hiveJoinType != JoinType.LEFTSEMI) { - joinRR = RowResolver.getCombinedRR(leftRR, rightRR); - } else { - joinRR = new RowResolver(); - if (!RowResolver.add(joinRR, leftRR)) { - LOG.warn("Duplicates detected when adding columns to RR: see previous message"); - } - } - - // 2. Construct ExpressionNodeDesc representing Join Condition + // 1. Construct ExpressionNodeDesc representing Join Condition RexNode calciteJoinCond = null; if (joinCond != null) { JoinTypeCheckCtx jCtx = new JoinTypeCheckCtx(leftRR, rightRR, hiveJoinType); @@ -1405,12 +1390,12 @@ private RelNode genJoinRelNode(RelNode leftRel, RelNode rightRel, JoinType hiveJ calciteJoinCond = cluster.getRexBuilder().makeLiteral(true); } - // 3. Validate that join condition is legal (i.e no function refering to + // 2. Validate that join condition is legal (i.e no function refering to // both sides of join, only equi join) // TODO: Join filter handling (only supported for OJ by runtime or is it // supported for IJ as well) - // 4. Construct Join Rel Node + // 3. Construct Join Rel Node and RowResolver for the new Join Node boolean leftSemiJoin = false; JoinRelType calciteJoinType; switch (hiveJoinType) { @@ -1433,6 +1418,8 @@ private RelNode genJoinRelNode(RelNode leftRel, RelNode rightRel, JoinType hiveJ break; } + RelNode topRel = null; + RowResolver topRR = null; if (leftSemiJoin) { List sysFieldList = new ArrayList(); List leftJoinKeys = new ArrayList(); @@ -1452,19 +1439,60 @@ private RelNode genJoinRelNode(RelNode leftRel, RelNode rightRel, JoinType hiveJ calciteJoinCond = HiveCalciteUtil.projectNonColumnEquiConditions( HiveRelFactories.HIVE_PROJECT_FACTORY, inputRels, leftJoinKeys, rightJoinKeys, 0, leftKeys, rightKeys); - - joinRel = HiveSemiJoin.getSemiJoin(cluster, cluster.traitSetOf(HiveRelNode.CONVENTION), + topRel = HiveSemiJoin.getSemiJoin(cluster, cluster.traitSetOf(HiveRelNode.CONVENTION), inputRels[0], inputRels[1], calciteJoinCond, ImmutableIntList.copyOf(leftKeys), ImmutableIntList.copyOf(rightKeys)); + + // Create join RR: we need to check whether we need to update left RR in case + // previous call to projectNonColumnEquiConditions updated it + if (inputRels[0] != leftRel) { + RowResolver newLeftRR = new RowResolver(); + if (!RowResolver.add(newLeftRR, leftRR)) { + LOG.warn("Duplicates detected when adding columns to RR: see previous message"); + } + for (int i = leftRel.getRowType().getFieldCount(); + i < inputRels[0].getRowType().getFieldCount(); i++) { + ColumnInfo oColInfo = new ColumnInfo( + SemanticAnalyzer.getColumnInternalName(i), + TypeConverter.convert(inputRels[0].getRowType().getFieldList().get(i).getType()), + null, false); + newLeftRR.put(oColInfo.getTabAlias(), oColInfo.getInternalName(), oColInfo); + } + + RowResolver joinRR = new RowResolver(); + if (!RowResolver.add(joinRR, newLeftRR)) { + LOG.warn("Duplicates detected when adding columns to RR: see previous message"); + } + relToHiveColNameCalcitePosMap.put(topRel, this.buildHiveToCalciteColumnMap(joinRR, topRel)); + relToHiveRR.put(topRel, joinRR); + + // Introduce top project operator to remove additional column(s) that have + // been introduced + List topFields = new ArrayList(); + List topFieldNames = new ArrayList(); + for (int i = 0; i < leftRel.getRowType().getFieldCount(); i++) { + final RelDataTypeField field = leftRel.getRowType().getFieldList().get(i); + topFields.add(leftRel.getCluster().getRexBuilder().makeInputRef(field.getType(), i)); + topFieldNames.add(field.getName()); + } + topRel = HiveRelFactories.HIVE_PROJECT_FACTORY.createProject(topRel, topFields, topFieldNames); + } + + topRR = new RowResolver(); + if (!RowResolver.add(topRR, leftRR)) { + LOG.warn("Duplicates detected when adding columns to RR: see previous message"); + } } else { - joinRel = HiveJoin.getJoin(cluster, leftRel, rightRel, calciteJoinCond, calciteJoinType, + topRel = HiveJoin.getJoin(cluster, leftRel, rightRel, calciteJoinCond, calciteJoinType, leftSemiJoin); + topRR = RowResolver.getCombinedRR(leftRR, rightRR); } - // 5. Add new JoinRel & its RR to the maps - relToHiveColNameCalcitePosMap.put(joinRel, this.buildHiveToCalciteColumnMap(joinRR, joinRel)); - relToHiveRR.put(joinRel, joinRR); - return joinRel; + // 4. Add new rel & its RR to the maps + relToHiveColNameCalcitePosMap.put(topRel, this.buildHiveToCalciteColumnMap(topRR, topRel)); + relToHiveRR.put(topRel, topRR); + + return topRel; } /** diff --git ql/src/test/queries/clientpositive/semijoin4.q ql/src/test/queries/clientpositive/semijoin4.q new file mode 100644 index 0000000..eabb037 --- /dev/null +++ ql/src/test/queries/clientpositive/semijoin4.q @@ -0,0 +1,24 @@ +set hive.mapred.mode=nonstrict; + +CREATE TABLE table_1 (int_col_1 INT, decimal3003_col_2 DECIMAL(30, 3), timestamp_col_3 TIMESTAMP, decimal0101_col_4 DECIMAL(1, 1), double_col_5 DOUBLE, boolean_col_6 BOOLEAN, timestamp_col_7 TIMESTAMP, varchar0098_col_8 VARCHAR(98), int_col_9 INT, timestamp_col_10 TIMESTAMP, decimal0903_col_11 DECIMAL(9, 3), int_col_12 INT, bigint_col_13 BIGINT, boolean_col_14 BOOLEAN, char0254_col_15 CHAR(254), boolean_col_16 BOOLEAN, smallint_col_17 SMALLINT, float_col_18 FLOAT, decimal2608_col_19 DECIMAL(26, 8), varchar0216_col_20 VARCHAR(216), string_col_21 STRING, timestamp_col_22 TIMESTAMP, double_col_23 DOUBLE, smallint_col_24 SMALLINT, float_col_25 FLOAT, decimal2016_col_26 DECIMAL(20, 16), string_col_27 STRING, decimal0202_col_28 DECIMAL(2, 2), boolean_col_29 BOOLEAN, decimal2020_col_30 DECIMAL(20, 20), float_col_31 FLOAT, boolean_col_32 BOOLEAN, varchar0148_col_33 VARCHAR(148), decimal2121_col_34 DECIMAL(21, 21), timestamp_col_35 TIMESTAMP, float_col_36 FLOAT, float_col_37 FLOAT, string_col_38 STRING, decimal3420_col_39 DECIMAL(34, 20), smallint_col_40 SMALLINT, decimal1408_col_41 DECIMAL(14, 8), string_col_42 STRING, decimal0902_col_43 DECIMAL(9, 2), varchar0204_col_44 VARCHAR(204), float_col_45 FLOAT, tinyint_col_46 TINYINT, double_col_47 DOUBLE, timestamp_col_48 TIMESTAMP, double_col_49 DOUBLE, timestamp_col_50 TIMESTAMP, decimal0704_col_51 DECIMAL(7, 4), int_col_52 INT, double_col_53 DOUBLE, int_col_54 INT, timestamp_col_55 TIMESTAMP, decimal0505_col_56 DECIMAL(5, 5), char0155_col_57 CHAR(155), double_col_58 DOUBLE, timestamp_col_59 TIMESTAMP, double_col_60 DOUBLE, float_col_61 FLOAT, char0249_col_62 CHAR(249), float_col_63 FLOAT, smallint_col_64 SMALLINT, decimal1309_col_65 DECIMAL(13, 9), timestamp_col_66 TIMESTAMP, boolean_col_67 BOOLEAN, tinyint_col_68 TINYINT, tinyint_col_69 TINYINT, double_col_70 DOUBLE, bigint_col_71 BIGINT, boolean_col_72 BOOLEAN, float_col_73 FLOAT, char0222_col_74 CHAR(222), boolean_col_75 BOOLEAN, string_col_76 STRING, decimal2612_col_77 DECIMAL(26, 12), bigint_col_78 BIGINT, char0128_col_79 CHAR(128), tinyint_col_80 TINYINT, boolean_col_81 BOOLEAN, int_col_82 INT, boolean_col_83 BOOLEAN, decimal2622_col_84 DECIMAL(26, 22), boolean_col_85 BOOLEAN, boolean_col_86 BOOLEAN, decimal0907_col_87 DECIMAL(9, 7)) +STORED AS orc; +CREATE TABLE table_18 (float_col_1 FLOAT, double_col_2 DOUBLE, decimal2518_col_3 DECIMAL(25, 18), boolean_col_4 BOOLEAN, bigint_col_5 BIGINT, boolean_col_6 BOOLEAN, boolean_col_7 BOOLEAN, char0035_col_8 CHAR(35), decimal2709_col_9 DECIMAL(27, 9), timestamp_col_10 TIMESTAMP, bigint_col_11 BIGINT, decimal3604_col_12 DECIMAL(36, 4), string_col_13 STRING, timestamp_col_14 TIMESTAMP, timestamp_col_15 TIMESTAMP, decimal1911_col_16 DECIMAL(19, 11), boolean_col_17 BOOLEAN, tinyint_col_18 TINYINT, timestamp_col_19 TIMESTAMP, timestamp_col_20 TIMESTAMP, tinyint_col_21 TINYINT, float_col_22 FLOAT, timestamp_col_23 TIMESTAMP) +STORED AS orc; + +explain +SELECT + COALESCE(498, + LEAD(COALESCE(-973, -684, 515)) OVER ( + PARTITION BY (t2.tinyint_col_21 + t1.smallint_col_24) + ORDER BY (t2.tinyint_col_21 + t1.smallint_col_24), + FLOOR(t1.double_col_60) DESC), + 524) AS int_col +FROM table_1 t1 INNER JOIN table_18 t2 +ON (((t2.tinyint_col_18) = (t1.bigint_col_13)) + AND ((t2.decimal2709_col_9) = (t1.decimal1309_col_65))) + AND ((t2.tinyint_col_21) = (t1.tinyint_col_46)) +WHERE (t2.tinyint_col_21) IN ( + SELECT COALESCE(-92, -994) AS int_col_3 + FROM table_1 tt1 INNER JOIN table_18 tt2 + ON (tt2.decimal1911_col_16) = (tt1.decimal1309_col_65) + WHERE (tt1.timestamp_col_66) = (tt2.timestamp_col_19)); diff --git ql/src/test/results/clientpositive/semijoin4.q.out ql/src/test/results/clientpositive/semijoin4.q.out new file mode 100644 index 0000000..7489de2 --- /dev/null +++ ql/src/test/results/clientpositive/semijoin4.q.out @@ -0,0 +1,257 @@ +PREHOOK: query: CREATE TABLE table_1 (int_col_1 INT, decimal3003_col_2 DECIMAL(30, 3), timestamp_col_3 TIMESTAMP, decimal0101_col_4 DECIMAL(1, 1), double_col_5 DOUBLE, boolean_col_6 BOOLEAN, timestamp_col_7 TIMESTAMP, varchar0098_col_8 VARCHAR(98), int_col_9 INT, timestamp_col_10 TIMESTAMP, decimal0903_col_11 DECIMAL(9, 3), int_col_12 INT, bigint_col_13 BIGINT, boolean_col_14 BOOLEAN, char0254_col_15 CHAR(254), boolean_col_16 BOOLEAN, smallint_col_17 SMALLINT, float_col_18 FLOAT, decimal2608_col_19 DECIMAL(26, 8), varchar0216_col_20 VARCHAR(216), string_col_21 STRING, timestamp_col_22 TIMESTAMP, double_col_23 DOUBLE, smallint_col_24 SMALLINT, float_col_25 FLOAT, decimal2016_col_26 DECIMAL(20, 16), string_col_27 STRING, decimal0202_col_28 DECIMAL(2, 2), boolean_col_29 BOOLEAN, decimal2020_col_30 DECIMAL(20, 20), float_col_31 FLOAT, boolean_col_32 BOOLEAN, varchar0148_col_33 VARCHAR(148), decimal2121_col_34 DECIMAL(21, 21), timestamp_col_35 TIMESTAMP, float_col_36 FLOAT, float_col_37 FLOAT, string_col_38 STRING, decimal3420_col_39 DECIMAL(34, 20), smallint_col_40 SMALLINT, decimal1408_col_41 DECIMAL(14, 8), string_col_42 STRING, decimal0902_col_43 DECIMAL(9, 2), varchar0204_col_44 VARCHAR(204), float_col_45 FLOAT, tinyint_col_46 TINYINT, double_col_47 DOUBLE, timestamp_col_48 TIMESTAMP, double_col_49 DOUBLE, timestamp_col_50 TIMESTAMP, decimal0704_col_51 DECIMAL(7, 4), int_col_52 INT, double_col_53 DOUBLE, int_col_54 INT, timestamp_col_55 TIMESTAMP, decimal0505_col_56 DECIMAL(5, 5), char0155_col_57 CHAR(155), double_col_58 DOUBLE, timestamp_col_59 TIMESTAMP, double_col_60 DOUBLE, float_col_61 FLOAT, char0249_col_62 CHAR(249), float_col_63 FLOAT, smallint_col_64 SMALLINT, decimal1309_col_65 DECIMAL(13, 9), timestamp_col_66 TIMESTAMP, boolean_col_67 BOOLEAN, tinyint_col_68 TINYINT, tinyint_col_69 TINYINT, double_col_70 DOUBLE, bigint_col_71 BIGINT, boolean_col_72 BOOLEAN, float_col_73 FLOAT, char0222_col_74 CHAR(222), boolean_col_75 BOOLEAN, string_col_76 STRING, decimal2612_col_77 DECIMAL(26, 12), bigint_col_78 BIGINT, char0128_col_79 CHAR(128), tinyint_col_80 TINYINT, boolean_col_81 BOOLEAN, int_col_82 INT, boolean_col_83 BOOLEAN, decimal2622_col_84 DECIMAL(26, 22), boolean_col_85 BOOLEAN, boolean_col_86 BOOLEAN, decimal0907_col_87 DECIMAL(9, 7)) +STORED AS orc +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@table_1 +POSTHOOK: query: CREATE TABLE table_1 (int_col_1 INT, decimal3003_col_2 DECIMAL(30, 3), timestamp_col_3 TIMESTAMP, decimal0101_col_4 DECIMAL(1, 1), double_col_5 DOUBLE, boolean_col_6 BOOLEAN, timestamp_col_7 TIMESTAMP, varchar0098_col_8 VARCHAR(98), int_col_9 INT, timestamp_col_10 TIMESTAMP, decimal0903_col_11 DECIMAL(9, 3), int_col_12 INT, bigint_col_13 BIGINT, boolean_col_14 BOOLEAN, char0254_col_15 CHAR(254), boolean_col_16 BOOLEAN, smallint_col_17 SMALLINT, float_col_18 FLOAT, decimal2608_col_19 DECIMAL(26, 8), varchar0216_col_20 VARCHAR(216), string_col_21 STRING, timestamp_col_22 TIMESTAMP, double_col_23 DOUBLE, smallint_col_24 SMALLINT, float_col_25 FLOAT, decimal2016_col_26 DECIMAL(20, 16), string_col_27 STRING, decimal0202_col_28 DECIMAL(2, 2), boolean_col_29 BOOLEAN, decimal2020_col_30 DECIMAL(20, 20), float_col_31 FLOAT, boolean_col_32 BOOLEAN, varchar0148_col_33 VARCHAR(148), decimal2121_col_34 DECIMAL(21, 21), timestamp_col_35 TIMESTAMP, float_col_36 FLOAT, float_col_37 FLOAT, string_col_38 STRING, decimal3420_col_39 DECIMAL(34, 20), smallint_col_40 SMALLINT, decimal1408_col_41 DECIMAL(14, 8), string_col_42 STRING, decimal0902_col_43 DECIMAL(9, 2), varchar0204_col_44 VARCHAR(204), float_col_45 FLOAT, tinyint_col_46 TINYINT, double_col_47 DOUBLE, timestamp_col_48 TIMESTAMP, double_col_49 DOUBLE, timestamp_col_50 TIMESTAMP, decimal0704_col_51 DECIMAL(7, 4), int_col_52 INT, double_col_53 DOUBLE, int_col_54 INT, timestamp_col_55 TIMESTAMP, decimal0505_col_56 DECIMAL(5, 5), char0155_col_57 CHAR(155), double_col_58 DOUBLE, timestamp_col_59 TIMESTAMP, double_col_60 DOUBLE, float_col_61 FLOAT, char0249_col_62 CHAR(249), float_col_63 FLOAT, smallint_col_64 SMALLINT, decimal1309_col_65 DECIMAL(13, 9), timestamp_col_66 TIMESTAMP, boolean_col_67 BOOLEAN, tinyint_col_68 TINYINT, tinyint_col_69 TINYINT, double_col_70 DOUBLE, bigint_col_71 BIGINT, boolean_col_72 BOOLEAN, float_col_73 FLOAT, char0222_col_74 CHAR(222), boolean_col_75 BOOLEAN, string_col_76 STRING, decimal2612_col_77 DECIMAL(26, 12), bigint_col_78 BIGINT, char0128_col_79 CHAR(128), tinyint_col_80 TINYINT, boolean_col_81 BOOLEAN, int_col_82 INT, boolean_col_83 BOOLEAN, decimal2622_col_84 DECIMAL(26, 22), boolean_col_85 BOOLEAN, boolean_col_86 BOOLEAN, decimal0907_col_87 DECIMAL(9, 7)) +STORED AS orc +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@table_1 +PREHOOK: query: CREATE TABLE table_18 (float_col_1 FLOAT, double_col_2 DOUBLE, decimal2518_col_3 DECIMAL(25, 18), boolean_col_4 BOOLEAN, bigint_col_5 BIGINT, boolean_col_6 BOOLEAN, boolean_col_7 BOOLEAN, char0035_col_8 CHAR(35), decimal2709_col_9 DECIMAL(27, 9), timestamp_col_10 TIMESTAMP, bigint_col_11 BIGINT, decimal3604_col_12 DECIMAL(36, 4), string_col_13 STRING, timestamp_col_14 TIMESTAMP, timestamp_col_15 TIMESTAMP, decimal1911_col_16 DECIMAL(19, 11), boolean_col_17 BOOLEAN, tinyint_col_18 TINYINT, timestamp_col_19 TIMESTAMP, timestamp_col_20 TIMESTAMP, tinyint_col_21 TINYINT, float_col_22 FLOAT, timestamp_col_23 TIMESTAMP) +STORED AS orc +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@table_18 +POSTHOOK: query: CREATE TABLE table_18 (float_col_1 FLOAT, double_col_2 DOUBLE, decimal2518_col_3 DECIMAL(25, 18), boolean_col_4 BOOLEAN, bigint_col_5 BIGINT, boolean_col_6 BOOLEAN, boolean_col_7 BOOLEAN, char0035_col_8 CHAR(35), decimal2709_col_9 DECIMAL(27, 9), timestamp_col_10 TIMESTAMP, bigint_col_11 BIGINT, decimal3604_col_12 DECIMAL(36, 4), string_col_13 STRING, timestamp_col_14 TIMESTAMP, timestamp_col_15 TIMESTAMP, decimal1911_col_16 DECIMAL(19, 11), boolean_col_17 BOOLEAN, tinyint_col_18 TINYINT, timestamp_col_19 TIMESTAMP, timestamp_col_20 TIMESTAMP, tinyint_col_21 TINYINT, float_col_22 FLOAT, timestamp_col_23 TIMESTAMP) +STORED AS orc +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@table_18 +PREHOOK: query: explain +SELECT + COALESCE(498, + LEAD(COALESCE(-973, -684, 515)) OVER ( + PARTITION BY (t2.tinyint_col_21 + t1.smallint_col_24) + ORDER BY (t2.tinyint_col_21 + t1.smallint_col_24), + FLOOR(t1.double_col_60) DESC), + 524) AS int_col +FROM table_1 t1 INNER JOIN table_18 t2 +ON (((t2.tinyint_col_18) = (t1.bigint_col_13)) + AND ((t2.decimal2709_col_9) = (t1.decimal1309_col_65))) + AND ((t2.tinyint_col_21) = (t1.tinyint_col_46)) +WHERE (t2.tinyint_col_21) IN ( + SELECT COALESCE(-92, -994) AS int_col_3 + FROM table_1 tt1 INNER JOIN table_18 tt2 + ON (tt2.decimal1911_col_16) = (tt1.decimal1309_col_65) + WHERE (tt1.timestamp_col_66) = (tt2.timestamp_col_19)) +PREHOOK: type: QUERY +POSTHOOK: query: explain +SELECT + COALESCE(498, + LEAD(COALESCE(-973, -684, 515)) OVER ( + PARTITION BY (t2.tinyint_col_21 + t1.smallint_col_24) + ORDER BY (t2.tinyint_col_21 + t1.smallint_col_24), + FLOOR(t1.double_col_60) DESC), + 524) AS int_col +FROM table_1 t1 INNER JOIN table_18 t2 +ON (((t2.tinyint_col_18) = (t1.bigint_col_13)) + AND ((t2.decimal2709_col_9) = (t1.decimal1309_col_65))) + AND ((t2.tinyint_col_21) = (t1.tinyint_col_46)) +WHERE (t2.tinyint_col_21) IN ( + SELECT COALESCE(-92, -994) AS int_col_3 + FROM table_1 tt1 INNER JOIN table_18 tt2 + ON (tt2.decimal1911_col_16) = (tt1.decimal1309_col_65) + WHERE (tt1.timestamp_col_66) = (tt2.timestamp_col_19)) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1, Stage-5 + Stage-3 depends on stages: Stage-2 + Stage-5 is a root stage + Stage-0 depends on stages: Stage-3 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: t1 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Filter Operator + predicate: ((tinyint_col_46 is not null and bigint_col_13 is not null) and decimal1309_col_65 is not null) (type: boolean) + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Select Operator + expressions: bigint_col_13 (type: bigint), smallint_col_24 (type: smallint), tinyint_col_46 (type: tinyint), double_col_60 (type: double), decimal1309_col_65 (type: decimal(13,9)) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: bigint), _col4 (type: decimal(27,9)), _col2 (type: tinyint) + sort order: +++ + Map-reduce partition columns: _col0 (type: bigint), _col4 (type: decimal(27,9)), _col2 (type: tinyint) + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + value expressions: _col1 (type: smallint), _col3 (type: double) + TableScan + alias: t2 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Filter Operator + predicate: (((decimal2709_col_9 is not null and tinyint_col_18 is not null) and (UDFToInteger(tinyint_col_21) = -92)) and UDFToInteger(tinyint_col_21) is not null) (type: boolean) + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Select Operator + expressions: decimal2709_col_9 (type: decimal(27,9)), tinyint_col_18 (type: tinyint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Reduce Output Operator + key expressions: UDFToLong(_col1) (type: bigint), _col0 (type: decimal(27,9)), -92 (type: tinyint) + sort order: +++ + Map-reduce partition columns: UDFToLong(_col1) (type: bigint), _col0 (type: decimal(27,9)), -92 (type: tinyint) + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Reduce Operator Tree: + Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: bigint), _col4 (type: decimal(27,9)), _col2 (type: tinyint) + 1 UDFToLong(_col1) (type: bigint), _col0 (type: decimal(27,9)), _col2 (type: tinyint) + outputColumnNames: _col1, _col3 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Select Operator + expressions: _col1 (type: smallint), _col3 (type: double), -92 (type: tinyint), -92 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + Reduce Output Operator + key expressions: _col3 (type: int) + sort order: + + Map-reduce partition columns: _col3 (type: int) + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + value expressions: _col0 (type: smallint), _col1 (type: double), _col2 (type: tinyint) + TableScan + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Reduce Operator Tree: + Join Operator + condition map: + Left Semi Join 0 to 1 + keys: + 0 _col3 (type: int) + 1 _col0 (type: int) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-3 + Map Reduce + Map Operator Tree: + TableScan + Reduce Output Operator + key expressions: (UDFToShort(_col2) + _col0) (type: smallint), floor(_col1) (type: bigint) + sort order: +- + Map-reduce partition columns: (UDFToShort(_col2) + _col0) (type: smallint) + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + value expressions: _col0 (type: smallint), _col1 (type: double), _col2 (type: tinyint) + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: double), VALUE._col2 (type: tinyint) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: smallint, _col1: double, _col2: tinyint + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: (UDFToShort(_col2) + _col0), floor(_col1)(DESC) + partition by: (UDFToShort(_col2) + _col0) + raw input shape: + window functions: + window function definition + alias: LEAD_window_0 + arguments: -973 + name: LEAD + window function: GenericUDAFLeadEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Select Operator + expressions: COALESCE(498,LEAD_window_0,524) (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-5 + Map Reduce + Map Operator Tree: + TableScan + alias: t1 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Filter Operator + predicate: (timestamp_col_66 is not null and decimal1309_col_65 is not null) (type: boolean) + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Select Operator + expressions: decimal1309_col_65 (type: decimal(13,9)), timestamp_col_66 (type: timestamp) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: decimal(19,11)), _col1 (type: timestamp) + sort order: ++ + Map-reduce partition columns: _col0 (type: decimal(19,11)), _col1 (type: timestamp) + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + TableScan + alias: t2 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Filter Operator + predicate: (timestamp_col_19 is not null and decimal1911_col_16 is not null) (type: boolean) + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Select Operator + expressions: decimal1911_col_16 (type: decimal(19,11)), timestamp_col_19 (type: timestamp) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: decimal(19,11)), _col1 (type: timestamp) + sort order: ++ + Map-reduce partition columns: _col0 (type: decimal(19,11)), _col1 (type: timestamp) + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Reduce Operator Tree: + Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: decimal(19,11)), _col1 (type: timestamp) + 1 _col0 (type: decimal(19,11)), _col1 (type: timestamp) + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + Group By Operator + keys: -92 (type: int) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink +