diff --git itests/hive-blobstore/src/test/results/clientpositive/insert_blobstore_to_blobstore.q.out itests/hive-blobstore/src/test/results/clientpositive/insert_blobstore_to_blobstore.q.out index 46518999358be9ec5191ad2c3b37f555efbec263..6b6a54be9eefa00e7e70fd613c62050b7a0d2494 100644 --- itests/hive-blobstore/src/test/results/clientpositive/insert_blobstore_to_blobstore.q.out +++ itests/hive-blobstore/src/test/results/clientpositive/insert_blobstore_to_blobstore.q.out @@ -60,6 +60,9 @@ POSTHOOK: query: INSERT OVERWRITE TABLE blobstore_table SELECT * FROM blobstore_ POSTHOOK: type: QUERY POSTHOOK: Input: default@blobstore_source POSTHOOK: Output: default@blobstore_table +POSTHOOK: Lineage: blobstore_table.a SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:a, type:string, comment:null), ] +POSTHOOK: Lineage: blobstore_table.b SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:b, type:string, comment:null), ] +POSTHOOK: Lineage: blobstore_table.c SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:c, type:double, comment:null), ] PREHOOK: query: SELECT COUNT(*) FROM blobstore_table PREHOOK: type: QUERY PREHOOK: Input: default@blobstore_table @@ -77,6 +80,9 @@ POSTHOOK: query: INSERT INTO TABLE blobstore_table SELECT * FROM blobstore_sourc POSTHOOK: type: QUERY POSTHOOK: Input: default@blobstore_source POSTHOOK: Output: default@blobstore_table +POSTHOOK: Lineage: blobstore_table.a SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:a, type:string, comment:null), ] +POSTHOOK: Lineage: blobstore_table.b SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:b, type:string, comment:null), ] +POSTHOOK: Lineage: blobstore_table.c SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:c, type:double, comment:null), ] PREHOOK: query: SELECT COUNT(*) FROM blobstore_table PREHOOK: type: QUERY PREHOOK: Input: default@blobstore_table diff --git itests/hive-blobstore/src/test/results/clientpositive/insert_empty_into_blobstore.q.out itests/hive-blobstore/src/test/results/clientpositive/insert_empty_into_blobstore.q.out index 8e5e09627b3a0f5c66a01fdd096b4b17b0b10689..ccd9ba54001370c96b77bbc1dcc0fd8e459efed4 100644 --- itests/hive-blobstore/src/test/results/clientpositive/insert_empty_into_blobstore.q.out +++ itests/hive-blobstore/src/test/results/clientpositive/insert_empty_into_blobstore.q.out @@ -108,6 +108,9 @@ POSTHOOK: query: INSERT INTO TABLE blobstore_list_bucketing PARTITION (pt='a', d POSTHOOK: type: QUERY POSTHOOK: Input: default@empty POSTHOOK: Output: default@blobstore_list_bucketing@pt=a/dt=a/hr=a +POSTHOOK: Lineage: blobstore_list_bucketing PARTITION(pt=a,dt=a,hr=a).dept SIMPLE [(empty)empty.FieldSchema(name:dept, type:string, comment:null), ] +POSTHOOK: Lineage: blobstore_list_bucketing PARTITION(pt=a,dt=a,hr=a).id SIMPLE [(empty)empty.FieldSchema(name:id, type:int, comment:null), ] +POSTHOOK: Lineage: blobstore_list_bucketing PARTITION(pt=a,dt=a,hr=a).name SIMPLE [(empty)empty.FieldSchema(name:name, type:string, comment:null), ] PREHOOK: query: SELECT COUNT(*) FROM blobstore_list_bucketing PREHOOK: type: QUERY PREHOOK: Input: default@blobstore_list_bucketing diff --git itests/hive-blobstore/src/test/results/clientpositive/insert_into_table.q.out itests/hive-blobstore/src/test/results/clientpositive/insert_into_table.q.out index 663a57274840a84a5f04777540d09d9f9bcfb5da..f50f4af817f000f6cc59133d5966899e79d67c3b 100644 --- itests/hive-blobstore/src/test/results/clientpositive/insert_into_table.q.out +++ itests/hive-blobstore/src/test/results/clientpositive/insert_into_table.q.out @@ -18,12 +18,14 @@ PREHOOK: Output: default@table1 POSTHOOK: query: INSERT INTO TABLE table1 VALUES (1) POSTHOOK: type: QUERY POSTHOOK: Output: default@table1 +POSTHOOK: Lineage: table1.id EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1, type:string, comment:), ] PREHOOK: query: INSERT INTO TABLE table1 VALUES (2) PREHOOK: type: QUERY PREHOOK: Output: default@table1 POSTHOOK: query: INSERT INTO TABLE table1 VALUES (2) POSTHOOK: type: QUERY POSTHOOK: Output: default@table1 +POSTHOOK: Lineage: table1.id EXPRESSION [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col1, type:string, comment:), ] PREHOOK: query: SELECT * FROM table1 PREHOOK: type: QUERY PREHOOK: Input: default@table1 diff --git itests/hive-blobstore/src/test/results/clientpositive/insert_overwrite_table.q.out itests/hive-blobstore/src/test/results/clientpositive/insert_overwrite_table.q.out index 8052fd86eefaff1fe7d024d3f58f6896e7223dd9..ba0e83d5623f6cd9d7ada998d47826883cb2aca4 100644 --- itests/hive-blobstore/src/test/results/clientpositive/insert_overwrite_table.q.out +++ itests/hive-blobstore/src/test/results/clientpositive/insert_overwrite_table.q.out @@ -18,6 +18,7 @@ PREHOOK: Output: default@table1 POSTHOOK: query: INSERT OVERWRITE TABLE table1 VALUES (1) POSTHOOK: type: QUERY POSTHOOK: Output: default@table1 +POSTHOOK: Lineage: table1.id EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1, type:string, comment:), ] PREHOOK: query: SELECT * FROM table1 PREHOOK: type: QUERY PREHOOK: Input: default@table1 @@ -33,6 +34,7 @@ PREHOOK: Output: default@table1 POSTHOOK: query: INSERT OVERWRITE TABLE table1 VALUES (2) POSTHOOK: type: QUERY POSTHOOK: Output: default@table1 +POSTHOOK: Lineage: table1.id EXPRESSION [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col1, type:string, comment:), ] PREHOOK: query: SELECT * FROM table1 PREHOOK: type: QUERY PREHOOK: Input: default@table1 diff --git itests/hive-blobstore/src/test/results/clientpositive/orc_format_nonpart.q.out itests/hive-blobstore/src/test/results/clientpositive/orc_format_nonpart.q.out index f4705687ff4b7b29d9bd6b81dfcb4065364c2965..954a62ac8953e4c2be4afff90095c7e4f2d13a10 100644 --- itests/hive-blobstore/src/test/results/clientpositive/orc_format_nonpart.q.out +++ itests/hive-blobstore/src/test/results/clientpositive/orc_format_nonpart.q.out @@ -56,6 +56,9 @@ SELECT * FROM blobstore_source POSTHOOK: type: QUERY POSTHOOK: Input: default@blobstore_source POSTHOOK: Output: default@orc_table +POSTHOOK: Lineage: orc_table.a EXPRESSION [(blobstore_source)blobstore_source.FieldSchema(name:a, type:string, comment:null), ] +POSTHOOK: Lineage: orc_table.b SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:b, type:string, comment:null), ] +POSTHOOK: Lineage: orc_table.value SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:c, type:double, comment:null), ] PREHOOK: query: SELECT * FROM orc_table PREHOOK: type: QUERY PREHOOK: Input: default@orc_table @@ -126,6 +129,9 @@ SELECT * FROM blobstore_source POSTHOOK: type: QUERY POSTHOOK: Input: default@blobstore_source POSTHOOK: Output: default@orc_table +POSTHOOK: Lineage: orc_table.a EXPRESSION [(blobstore_source)blobstore_source.FieldSchema(name:a, type:string, comment:null), ] +POSTHOOK: Lineage: orc_table.b SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:b, type:string, comment:null), ] +POSTHOOK: Lineage: orc_table.value SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:c, type:double, comment:null), ] PREHOOK: query: SELECT * FROM orc_table PREHOOK: type: QUERY PREHOOK: Input: default@orc_table diff --git itests/hive-blobstore/src/test/results/clientpositive/orc_format_part.q.out itests/hive-blobstore/src/test/results/clientpositive/orc_format_part.q.out index 1ef98104c5a711ef4b6553ca06fccf9003b99361..7b2561358a95285fbfaaa6d196e35a0bea62ce85 100644 --- itests/hive-blobstore/src/test/results/clientpositive/orc_format_part.q.out +++ itests/hive-blobstore/src/test/results/clientpositive/orc_format_part.q.out @@ -242,6 +242,14 @@ WHERE SUBSTR(run_date,1,6)='201209' AND game_id=39 AND event_name='hq_change' POSTHOOK: type: QUERY POSTHOOK: Input: default@src_events POSTHOOK: Output: default@orc_events@run_date=201209/game_id=39/event_name=hq_change +POSTHOOK: Lineage: orc_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).event_data SIMPLE [(src_events)src_events.FieldSchema(name:event_data, type:string, comment:null), ] +POSTHOOK: Lineage: orc_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).full_uid SIMPLE [(src_events)src_events.FieldSchema(name:full_uid, type:bigint, comment:null), ] +POSTHOOK: Lineage: orc_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).log_id SIMPLE [(src_events)src_events.FieldSchema(name:log_id, type:bigint, comment:null), ] +POSTHOOK: Lineage: orc_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).session_id SIMPLE [(src_events)src_events.FieldSchema(name:session_id, type:string, comment:null), ] +POSTHOOK: Lineage: orc_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).time SIMPLE [(src_events)src_events.FieldSchema(name:time, type:bigint, comment:null), ] +POSTHOOK: Lineage: orc_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).type SIMPLE [(src_events)src_events.FieldSchema(name:type, type:int, comment:null), ] +POSTHOOK: Lineage: orc_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).uid SIMPLE [(src_events)src_events.FieldSchema(name:uid, type:bigint, comment:null), ] +POSTHOOK: Lineage: orc_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).user_id SIMPLE [(src_events)src_events.FieldSchema(name:user_id, type:bigint, comment:null), ] PREHOOK: query: SELECT COUNT(*) FROM orc_events PREHOOK: type: QUERY PREHOOK: Input: default@orc_events @@ -263,6 +271,14 @@ WHERE SUBSTR(run_date,1,6)='201209' AND game_id=39 AND event_name='hq_change' POSTHOOK: type: QUERY POSTHOOK: Input: default@src_events POSTHOOK: Output: default@orc_events@run_date=201209/game_id=39/event_name=hq_change +POSTHOOK: Lineage: orc_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).event_data SIMPLE [(src_events)src_events.FieldSchema(name:event_data, type:string, comment:null), ] +POSTHOOK: Lineage: orc_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).full_uid SIMPLE [(src_events)src_events.FieldSchema(name:full_uid, type:bigint, comment:null), ] +POSTHOOK: Lineage: orc_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).log_id SIMPLE [(src_events)src_events.FieldSchema(name:log_id, type:bigint, comment:null), ] +POSTHOOK: Lineage: orc_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).session_id SIMPLE [(src_events)src_events.FieldSchema(name:session_id, type:string, comment:null), ] +POSTHOOK: Lineage: orc_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).time SIMPLE [(src_events)src_events.FieldSchema(name:time, type:bigint, comment:null), ] +POSTHOOK: Lineage: orc_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).type SIMPLE [(src_events)src_events.FieldSchema(name:type, type:int, comment:null), ] +POSTHOOK: Lineage: orc_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).uid SIMPLE [(src_events)src_events.FieldSchema(name:uid, type:bigint, comment:null), ] +POSTHOOK: Lineage: orc_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).user_id SIMPLE [(src_events)src_events.FieldSchema(name:user_id, type:bigint, comment:null), ] PREHOOK: query: SELECT COUNT(*) FROM orc_events PREHOOK: type: QUERY PREHOOK: Input: default@orc_events diff --git itests/hive-blobstore/src/test/results/clientpositive/orc_nonstd_partitions_loc.q.out itests/hive-blobstore/src/test/results/clientpositive/orc_nonstd_partitions_loc.q.out index 9de4190072cc779b34e556a0e774ce7403324525..1201ce2107cdd428eb481f48ea5444a3ac71d68b 100644 --- itests/hive-blobstore/src/test/results/clientpositive/orc_nonstd_partitions_loc.q.out +++ itests/hive-blobstore/src/test/results/clientpositive/orc_nonstd_partitions_loc.q.out @@ -167,6 +167,14 @@ WHERE SUBSTR(run_date,1,6)='201211' POSTHOOK: type: QUERY POSTHOOK: Input: default@src_events POSTHOOK: Output: default@orc_events@run_date=201211/game_id=39/event_name=hq_change +POSTHOOK: Lineage: orc_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).event_data SIMPLE [(src_events)src_events.FieldSchema(name:event_data, type:string, comment:null), ] +POSTHOOK: Lineage: orc_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).full_uid SIMPLE [(src_events)src_events.FieldSchema(name:full_uid, type:bigint, comment:null), ] +POSTHOOK: Lineage: orc_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).log_id SIMPLE [(src_events)src_events.FieldSchema(name:log_id, type:bigint, comment:null), ] +POSTHOOK: Lineage: orc_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).session_id SIMPLE [(src_events)src_events.FieldSchema(name:session_id, type:string, comment:null), ] +POSTHOOK: Lineage: orc_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).time SIMPLE [(src_events)src_events.FieldSchema(name:time, type:bigint, comment:null), ] +POSTHOOK: Lineage: orc_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).type SIMPLE [(src_events)src_events.FieldSchema(name:type, type:int, comment:null), ] +POSTHOOK: Lineage: orc_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).uid SIMPLE [(src_events)src_events.FieldSchema(name:uid, type:bigint, comment:null), ] +POSTHOOK: Lineage: orc_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).user_id SIMPLE [(src_events)src_events.FieldSchema(name:user_id, type:bigint, comment:null), ] PREHOOK: query: SHOW PARTITIONS orc_events PREHOOK: type: SHOWPARTITIONS PREHOOK: Input: default@orc_events @@ -198,6 +206,14 @@ WHERE SUBSTR(run_date,1,6)='201211' POSTHOOK: type: QUERY POSTHOOK: Input: default@src_events POSTHOOK: Output: default@orc_events@run_date=201211/game_id=39/event_name=hq_change +POSTHOOK: Lineage: orc_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).event_data SIMPLE [(src_events)src_events.FieldSchema(name:event_data, type:string, comment:null), ] +POSTHOOK: Lineage: orc_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).full_uid SIMPLE [(src_events)src_events.FieldSchema(name:full_uid, type:bigint, comment:null), ] +POSTHOOK: Lineage: orc_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).log_id SIMPLE [(src_events)src_events.FieldSchema(name:log_id, type:bigint, comment:null), ] +POSTHOOK: Lineage: orc_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).session_id SIMPLE [(src_events)src_events.FieldSchema(name:session_id, type:string, comment:null), ] +POSTHOOK: Lineage: orc_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).time SIMPLE [(src_events)src_events.FieldSchema(name:time, type:bigint, comment:null), ] +POSTHOOK: Lineage: orc_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).type SIMPLE [(src_events)src_events.FieldSchema(name:type, type:int, comment:null), ] +POSTHOOK: Lineage: orc_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).uid SIMPLE [(src_events)src_events.FieldSchema(name:uid, type:bigint, comment:null), ] +POSTHOOK: Lineage: orc_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).user_id SIMPLE [(src_events)src_events.FieldSchema(name:user_id, type:bigint, comment:null), ] PREHOOK: query: SHOW PARTITIONS orc_events PREHOOK: type: SHOWPARTITIONS PREHOOK: Input: default@orc_events @@ -367,6 +383,14 @@ WHERE SUBSTR(run_date,1,6)='201209' POSTHOOK: type: QUERY POSTHOOK: Input: default@src_events POSTHOOK: Output: default@orc_events@run_date=201207/game_id=39/event_name=hq_change +POSTHOOK: Lineage: orc_events PARTITION(run_date=201207,game_id=39,event_name=hq_change).event_data SIMPLE [(src_events)src_events.FieldSchema(name:event_data, type:string, comment:null), ] +POSTHOOK: Lineage: orc_events PARTITION(run_date=201207,game_id=39,event_name=hq_change).full_uid SIMPLE [(src_events)src_events.FieldSchema(name:full_uid, type:bigint, comment:null), ] +POSTHOOK: Lineage: orc_events PARTITION(run_date=201207,game_id=39,event_name=hq_change).log_id SIMPLE [(src_events)src_events.FieldSchema(name:log_id, type:bigint, comment:null), ] +POSTHOOK: Lineage: orc_events PARTITION(run_date=201207,game_id=39,event_name=hq_change).session_id SIMPLE [(src_events)src_events.FieldSchema(name:session_id, type:string, comment:null), ] +POSTHOOK: Lineage: orc_events PARTITION(run_date=201207,game_id=39,event_name=hq_change).time SIMPLE [(src_events)src_events.FieldSchema(name:time, type:bigint, comment:null), ] +POSTHOOK: Lineage: orc_events PARTITION(run_date=201207,game_id=39,event_name=hq_change).type SIMPLE [(src_events)src_events.FieldSchema(name:type, type:int, comment:null), ] +POSTHOOK: Lineage: orc_events PARTITION(run_date=201207,game_id=39,event_name=hq_change).uid SIMPLE [(src_events)src_events.FieldSchema(name:uid, type:bigint, comment:null), ] +POSTHOOK: Lineage: orc_events PARTITION(run_date=201207,game_id=39,event_name=hq_change).user_id SIMPLE [(src_events)src_events.FieldSchema(name:user_id, type:bigint, comment:null), ] PREHOOK: query: SHOW PARTITIONS orc_events PREHOOK: type: SHOWPARTITIONS PREHOOK: Input: default@orc_events diff --git itests/hive-blobstore/src/test/results/clientpositive/rcfile_format_nonpart.q.out itests/hive-blobstore/src/test/results/clientpositive/rcfile_format_nonpart.q.out index 44a1f11fa2c7bd32c636b9bad4e4e35d79989dcf..de5104bb51b001b5fa457806ed7f0d83c8926985 100644 --- itests/hive-blobstore/src/test/results/clientpositive/rcfile_format_nonpart.q.out +++ itests/hive-blobstore/src/test/results/clientpositive/rcfile_format_nonpart.q.out @@ -56,6 +56,9 @@ SELECT * FROM blobstore_source POSTHOOK: type: QUERY POSTHOOK: Input: default@blobstore_source POSTHOOK: Output: default@rcfile_table +POSTHOOK: Lineage: rcfile_table.a EXPRESSION [(blobstore_source)blobstore_source.FieldSchema(name:a, type:string, comment:null), ] +POSTHOOK: Lineage: rcfile_table.b SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:b, type:string, comment:null), ] +POSTHOOK: Lineage: rcfile_table.value SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:c, type:double, comment:null), ] PREHOOK: query: SELECT * FROM rcfile_table PREHOOK: type: QUERY PREHOOK: Input: default@rcfile_table @@ -126,6 +129,9 @@ SELECT * FROM blobstore_source POSTHOOK: type: QUERY POSTHOOK: Input: default@blobstore_source POSTHOOK: Output: default@rcfile_table +POSTHOOK: Lineage: rcfile_table.a EXPRESSION [(blobstore_source)blobstore_source.FieldSchema(name:a, type:string, comment:null), ] +POSTHOOK: Lineage: rcfile_table.b SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:b, type:string, comment:null), ] +POSTHOOK: Lineage: rcfile_table.value SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:c, type:double, comment:null), ] PREHOOK: query: SELECT * FROM rcfile_table PREHOOK: type: QUERY PREHOOK: Input: default@rcfile_table diff --git itests/hive-blobstore/src/test/results/clientpositive/rcfile_format_part.q.out itests/hive-blobstore/src/test/results/clientpositive/rcfile_format_part.q.out index defca3bb78d4d97a327c389650d256a2cd880899..24fc5258043465cc6a00a271ca3e797884063aaf 100644 --- itests/hive-blobstore/src/test/results/clientpositive/rcfile_format_part.q.out +++ itests/hive-blobstore/src/test/results/clientpositive/rcfile_format_part.q.out @@ -242,6 +242,14 @@ WHERE SUBSTR(run_date,1,6)='201209' AND game_id=39 AND event_name='hq_change' POSTHOOK: type: QUERY POSTHOOK: Input: default@src_events POSTHOOK: Output: default@rcfile_events@run_date=201209/game_id=39/event_name=hq_change +POSTHOOK: Lineage: rcfile_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).event_data SIMPLE [(src_events)src_events.FieldSchema(name:event_data, type:string, comment:null), ] +POSTHOOK: Lineage: rcfile_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).full_uid SIMPLE [(src_events)src_events.FieldSchema(name:full_uid, type:bigint, comment:null), ] +POSTHOOK: Lineage: rcfile_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).log_id SIMPLE [(src_events)src_events.FieldSchema(name:log_id, type:bigint, comment:null), ] +POSTHOOK: Lineage: rcfile_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).session_id SIMPLE [(src_events)src_events.FieldSchema(name:session_id, type:string, comment:null), ] +POSTHOOK: Lineage: rcfile_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).time SIMPLE [(src_events)src_events.FieldSchema(name:time, type:bigint, comment:null), ] +POSTHOOK: Lineage: rcfile_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).type SIMPLE [(src_events)src_events.FieldSchema(name:type, type:int, comment:null), ] +POSTHOOK: Lineage: rcfile_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).uid SIMPLE [(src_events)src_events.FieldSchema(name:uid, type:bigint, comment:null), ] +POSTHOOK: Lineage: rcfile_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).user_id SIMPLE [(src_events)src_events.FieldSchema(name:user_id, type:bigint, comment:null), ] PREHOOK: query: SELECT COUNT(*) FROM rcfile_events PREHOOK: type: QUERY PREHOOK: Input: default@rcfile_events @@ -263,6 +271,14 @@ WHERE SUBSTR(run_date,1,6)='201209' AND game_id=39 AND event_name='hq_change' POSTHOOK: type: QUERY POSTHOOK: Input: default@src_events POSTHOOK: Output: default@rcfile_events@run_date=201209/game_id=39/event_name=hq_change +POSTHOOK: Lineage: rcfile_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).event_data SIMPLE [(src_events)src_events.FieldSchema(name:event_data, type:string, comment:null), ] +POSTHOOK: Lineage: rcfile_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).full_uid SIMPLE [(src_events)src_events.FieldSchema(name:full_uid, type:bigint, comment:null), ] +POSTHOOK: Lineage: rcfile_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).log_id SIMPLE [(src_events)src_events.FieldSchema(name:log_id, type:bigint, comment:null), ] +POSTHOOK: Lineage: rcfile_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).session_id SIMPLE [(src_events)src_events.FieldSchema(name:session_id, type:string, comment:null), ] +POSTHOOK: Lineage: rcfile_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).time SIMPLE [(src_events)src_events.FieldSchema(name:time, type:bigint, comment:null), ] +POSTHOOK: Lineage: rcfile_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).type SIMPLE [(src_events)src_events.FieldSchema(name:type, type:int, comment:null), ] +POSTHOOK: Lineage: rcfile_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).uid SIMPLE [(src_events)src_events.FieldSchema(name:uid, type:bigint, comment:null), ] +POSTHOOK: Lineage: rcfile_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).user_id SIMPLE [(src_events)src_events.FieldSchema(name:user_id, type:bigint, comment:null), ] PREHOOK: query: SELECT COUNT(*) FROM rcfile_events PREHOOK: type: QUERY PREHOOK: Input: default@rcfile_events diff --git itests/hive-blobstore/src/test/results/clientpositive/rcfile_nonstd_partitions_loc.q.out itests/hive-blobstore/src/test/results/clientpositive/rcfile_nonstd_partitions_loc.q.out index 5db9c7ef90c81524eae3eb6ec5346321e38b2a9e..6bcfe41262dee389e854aa399dcc854a2b16599b 100644 --- itests/hive-blobstore/src/test/results/clientpositive/rcfile_nonstd_partitions_loc.q.out +++ itests/hive-blobstore/src/test/results/clientpositive/rcfile_nonstd_partitions_loc.q.out @@ -167,6 +167,14 @@ WHERE SUBSTR(run_date,1,6)='201211' POSTHOOK: type: QUERY POSTHOOK: Input: default@src_events POSTHOOK: Output: default@rcfile_events@run_date=201211/game_id=39/event_name=hq_change +POSTHOOK: Lineage: rcfile_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).event_data SIMPLE [(src_events)src_events.FieldSchema(name:event_data, type:string, comment:null), ] +POSTHOOK: Lineage: rcfile_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).full_uid SIMPLE [(src_events)src_events.FieldSchema(name:full_uid, type:bigint, comment:null), ] +POSTHOOK: Lineage: rcfile_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).log_id SIMPLE [(src_events)src_events.FieldSchema(name:log_id, type:bigint, comment:null), ] +POSTHOOK: Lineage: rcfile_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).session_id SIMPLE [(src_events)src_events.FieldSchema(name:session_id, type:string, comment:null), ] +POSTHOOK: Lineage: rcfile_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).time SIMPLE [(src_events)src_events.FieldSchema(name:time, type:bigint, comment:null), ] +POSTHOOK: Lineage: rcfile_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).type SIMPLE [(src_events)src_events.FieldSchema(name:type, type:int, comment:null), ] +POSTHOOK: Lineage: rcfile_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).uid SIMPLE [(src_events)src_events.FieldSchema(name:uid, type:bigint, comment:null), ] +POSTHOOK: Lineage: rcfile_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).user_id SIMPLE [(src_events)src_events.FieldSchema(name:user_id, type:bigint, comment:null), ] PREHOOK: query: SHOW PARTITIONS rcfile_events PREHOOK: type: SHOWPARTITIONS PREHOOK: Input: default@rcfile_events @@ -198,6 +206,14 @@ WHERE SUBSTR(run_date,1,6)='201211' POSTHOOK: type: QUERY POSTHOOK: Input: default@src_events POSTHOOK: Output: default@rcfile_events@run_date=201211/game_id=39/event_name=hq_change +POSTHOOK: Lineage: rcfile_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).event_data SIMPLE [(src_events)src_events.FieldSchema(name:event_data, type:string, comment:null), ] +POSTHOOK: Lineage: rcfile_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).full_uid SIMPLE [(src_events)src_events.FieldSchema(name:full_uid, type:bigint, comment:null), ] +POSTHOOK: Lineage: rcfile_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).log_id SIMPLE [(src_events)src_events.FieldSchema(name:log_id, type:bigint, comment:null), ] +POSTHOOK: Lineage: rcfile_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).session_id SIMPLE [(src_events)src_events.FieldSchema(name:session_id, type:string, comment:null), ] +POSTHOOK: Lineage: rcfile_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).time SIMPLE [(src_events)src_events.FieldSchema(name:time, type:bigint, comment:null), ] +POSTHOOK: Lineage: rcfile_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).type SIMPLE [(src_events)src_events.FieldSchema(name:type, type:int, comment:null), ] +POSTHOOK: Lineage: rcfile_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).uid SIMPLE [(src_events)src_events.FieldSchema(name:uid, type:bigint, comment:null), ] +POSTHOOK: Lineage: rcfile_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).user_id SIMPLE [(src_events)src_events.FieldSchema(name:user_id, type:bigint, comment:null), ] PREHOOK: query: SHOW PARTITIONS rcfile_events PREHOOK: type: SHOWPARTITIONS PREHOOK: Input: default@rcfile_events @@ -387,6 +403,14 @@ WHERE SUBSTR(run_date,1,6)='201209' POSTHOOK: type: QUERY POSTHOOK: Input: default@src_events POSTHOOK: Output: default@rcfile_events@run_date=201207/game_id=39/event_name=hq_change +POSTHOOK: Lineage: rcfile_events PARTITION(run_date=201207,game_id=39,event_name=hq_change).event_data SIMPLE [(src_events)src_events.FieldSchema(name:event_data, type:string, comment:null), ] +POSTHOOK: Lineage: rcfile_events PARTITION(run_date=201207,game_id=39,event_name=hq_change).full_uid SIMPLE [(src_events)src_events.FieldSchema(name:full_uid, type:bigint, comment:null), ] +POSTHOOK: Lineage: rcfile_events PARTITION(run_date=201207,game_id=39,event_name=hq_change).log_id SIMPLE [(src_events)src_events.FieldSchema(name:log_id, type:bigint, comment:null), ] +POSTHOOK: Lineage: rcfile_events PARTITION(run_date=201207,game_id=39,event_name=hq_change).session_id SIMPLE [(src_events)src_events.FieldSchema(name:session_id, type:string, comment:null), ] +POSTHOOK: Lineage: rcfile_events PARTITION(run_date=201207,game_id=39,event_name=hq_change).time SIMPLE [(src_events)src_events.FieldSchema(name:time, type:bigint, comment:null), ] +POSTHOOK: Lineage: rcfile_events PARTITION(run_date=201207,game_id=39,event_name=hq_change).type SIMPLE [(src_events)src_events.FieldSchema(name:type, type:int, comment:null), ] +POSTHOOK: Lineage: rcfile_events PARTITION(run_date=201207,game_id=39,event_name=hq_change).uid SIMPLE [(src_events)src_events.FieldSchema(name:uid, type:bigint, comment:null), ] +POSTHOOK: Lineage: rcfile_events PARTITION(run_date=201207,game_id=39,event_name=hq_change).user_id SIMPLE [(src_events)src_events.FieldSchema(name:user_id, type:bigint, comment:null), ] PREHOOK: query: SHOW PARTITIONS rcfile_events PREHOOK: type: SHOWPARTITIONS PREHOOK: Input: default@rcfile_events diff --git itests/hive-blobstore/src/test/results/clientpositive/zero_rows_blobstore.q.out itests/hive-blobstore/src/test/results/clientpositive/zero_rows_blobstore.q.out index 590f947add9b34d3adb05a7e22b0eb7d011c9a0d..20f8311bda6d297da809a2d00f7196f929b61b95 100644 --- itests/hive-blobstore/src/test/results/clientpositive/zero_rows_blobstore.q.out +++ itests/hive-blobstore/src/test/results/clientpositive/zero_rows_blobstore.q.out @@ -63,6 +63,7 @@ POSTHOOK: query: INSERT OVERWRITE TABLE blobstore_target SELECT key FROM blobsto POSTHOOK: type: QUERY POSTHOOK: Input: default@blobstore_source POSTHOOK: Output: default@blobstore_target +POSTHOOK: Lineage: blobstore_target.key SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:key, type:int, comment:null), ] PREHOOK: query: SELECT COUNT(*) FROM blobstore_target PREHOOK: type: QUERY PREHOOK: Input: default@blobstore_target @@ -80,6 +81,7 @@ POSTHOOK: query: INSERT OVERWRITE TABLE blobstore_target SELECT key FROM blobsto POSTHOOK: type: QUERY POSTHOOK: Input: default@blobstore_source POSTHOOK: Output: default@blobstore_target +POSTHOOK: Lineage: blobstore_target.key SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:key, type:int, comment:null), ] PREHOOK: query: SELECT COUNT(*) FROM blobstore_target PREHOOK: type: QUERY PREHOOK: Input: default@blobstore_target diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java index 88bf82999953346bc92dac17c07373e6360be232..da153e36d2d0a4e0de1a68e8f26ead963a2317a6 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java @@ -115,6 +115,8 @@ import org.apache.hadoop.hive.ql.plan.TableDesc; import org.apache.hadoop.hive.ql.plan.TableScanDesc; import org.apache.hadoop.hive.ql.plan.TezWork; +import org.apache.hadoop.hive.ql.session.LineageState; +import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hadoop.hive.serde2.SerDeException; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; @@ -1679,12 +1681,15 @@ protected static MoveWork mergeMovePaths(Path condInputPath, MoveWork linkedMove LoadFileDesc fileDesc = null; LoadTableDesc tableDesc = null; + LineageState lineageState = SessionState.get().getLineageState(); if (linkedMoveWork.getLoadFileWork() != null) { fileDesc = new LoadFileDesc(linkedMoveWork.getLoadFileWork()); fileDesc.setSourcePath(condInputPath); + lineageState.updateDirToOpMap(condInputPath, linkedMoveWork.getLoadFileWork().getSourcePath()); } else if (linkedMoveWork.getLoadTableWork() != null) { tableDesc = new LoadTableDesc(linkedMoveWork.getLoadTableWork()); tableDesc.setSourcePath(condInputPath); + lineageState.updateDirToOpMap(condInputPath, linkedMoveWork.getLoadTableWork().getSourcePath()); } else { throw new IllegalArgumentException("Merging a path with a MoveWork with multi-files work is not allowed."); } diff --git ql/src/java/org/apache/hadoop/hive/ql/session/LineageState.java ql/src/java/org/apache/hadoop/hive/ql/session/LineageState.java index 0f9506376c283ec6e6ab659b20a5718807268969..2a606a4a3a68a631679da2f5c387b2b75135e91b 100644 --- ql/src/java/org/apache/hadoop/hive/ql/session/LineageState.java +++ ql/src/java/org/apache/hadoop/hive/ql/session/LineageState.java @@ -76,6 +76,13 @@ public void mapDirToOp(Path dir, Operator fop) { dirToFop.put(dir, fop); } + public void updateDirToOpMap(Path newPath, Path oldPath) { + Operator op = dirToFop.remove(oldPath); + if (op != null) { + dirToFop.put(newPath, op); + } + } + /** * Set the lineage information for the associated directory. * diff --git ql/src/test/org/apache/hadoop/hive/ql/optimizer/TestGenMapRedUtilsCreateConditionalTask.java ql/src/test/org/apache/hadoop/hive/ql/optimizer/TestGenMapRedUtilsCreateConditionalTask.java index 68ccda9ef61744e369638ceec5c083c77524c2db..e7ce2345cd558437e1d5fce7ea14473f63bd1b89 100644 --- ql/src/test/org/apache/hadoop/hive/ql/optimizer/TestGenMapRedUtilsCreateConditionalTask.java +++ ql/src/test/org/apache/hadoop/hive/ql/optimizer/TestGenMapRedUtilsCreateConditionalTask.java @@ -26,6 +26,7 @@ import org.apache.hadoop.hive.ql.io.HiveOutputFormat; import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.plan.*; +import org.apache.hadoop.hive.ql.session.SessionState; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; @@ -53,6 +54,7 @@ public static void initializeSessionState() { @Before public void setUp() { dummyMRTask = new MapRedTask(); + SessionState.start(hiveConf); } @Test