diff --git itests/hive-blobstore/src/test/results/clientpositive/insert_overwrite_dynamic_partitions_move_only.q.out itests/hive-blobstore/src/test/results/clientpositive/insert_overwrite_dynamic_partitions_move_only.q.out index 530c0368caa3aa2acf0ff4d4c8a52091c3ae00c2..cc1d018596732c1467baa5602b9d33f37c140013 100644 --- itests/hive-blobstore/src/test/results/clientpositive/insert_overwrite_dynamic_partitions_move_only.q.out +++ itests/hive-blobstore/src/test/results/clientpositive/insert_overwrite_dynamic_partitions_move_only.q.out @@ -50,6 +50,16 @@ POSTHOOK: Output: default@s3_table_move@reported_date=2017-04-10/product_id=1000 POSTHOOK: Output: default@s3_table_move@reported_date=2017-04-10/product_id=10002 POSTHOOK: Output: default@s3_table_move@reported_date=2017-04-12/product_id=10001 POSTHOOK: Output: default@s3_table_move@reported_date=2017-04-12/product_id=10002 +POSTHOOK: Lineage: s3_table_move PARTITION(reported_date=2017-04-10,product_id=10000).event_name SIMPLE [(tmp_table_move)t.FieldSchema(name:name, type:string, comment:null), ] +POSTHOOK: Lineage: s3_table_move PARTITION(reported_date=2017-04-10,product_id=10000).user_id SIMPLE [(tmp_table_move)t.FieldSchema(name:id, type:string, comment:null), ] +POSTHOOK: Lineage: s3_table_move PARTITION(reported_date=2017-04-10,product_id=10001).event_name SIMPLE [(tmp_table_move)t.FieldSchema(name:name, type:string, comment:null), ] +POSTHOOK: Lineage: s3_table_move PARTITION(reported_date=2017-04-10,product_id=10001).user_id SIMPLE [(tmp_table_move)t.FieldSchema(name:id, type:string, comment:null), ] +POSTHOOK: Lineage: s3_table_move PARTITION(reported_date=2017-04-10,product_id=10002).event_name SIMPLE [(tmp_table_move)t.FieldSchema(name:name, type:string, comment:null), ] +POSTHOOK: Lineage: s3_table_move PARTITION(reported_date=2017-04-10,product_id=10002).user_id SIMPLE [(tmp_table_move)t.FieldSchema(name:id, type:string, comment:null), ] +POSTHOOK: Lineage: s3_table_move PARTITION(reported_date=2017-04-12,product_id=10001).event_name SIMPLE [(tmp_table_move)t.FieldSchema(name:name, type:string, comment:null), ] +POSTHOOK: Lineage: s3_table_move PARTITION(reported_date=2017-04-12,product_id=10001).user_id SIMPLE [(tmp_table_move)t.FieldSchema(name:id, type:string, comment:null), ] +POSTHOOK: Lineage: s3_table_move PARTITION(reported_date=2017-04-12,product_id=10002).event_name SIMPLE [(tmp_table_move)t.FieldSchema(name:name, type:string, comment:null), ] +POSTHOOK: Lineage: s3_table_move PARTITION(reported_date=2017-04-12,product_id=10002).user_id SIMPLE [(tmp_table_move)t.FieldSchema(name:id, type:string, comment:null), ] PREHOOK: query: select * from s3_table_move order by user_id PREHOOK: type: QUERY PREHOOK: Input: default@s3_table_move diff --git ql/src/java/org/apache/hadoop/hive/ql/session/LineageState.java ql/src/java/org/apache/hadoop/hive/ql/session/LineageState.java index 2a606a4a3a68a631679da2f5c387b2b75135e91b..e2f2a68ff9b002ede28b5306c3714e63482eb7d1 100644 --- ql/src/java/org/apache/hadoop/hive/ql/session/LineageState.java +++ ql/src/java/org/apache/hadoop/hive/ql/session/LineageState.java @@ -76,8 +76,15 @@ public void mapDirToOp(Path dir, Operator fop) { dirToFop.put(dir, fop); } + /** + * Update the path of the captured lineage information in case the + * conditional input path and the linked MoveWork were merged into one MoveWork. + * This should only happen for Blobstore systems with optimization turned on. + * @param newPath conditional input path + * @param oldPath path of the old linked MoveWork + */ public void updateDirToOpMap(Path newPath, Path oldPath) { - Operator op = dirToFop.remove(oldPath); + Operator op = dirToFop.get(oldPath); if (op != null) { dirToFop.put(newPath, op); }