commit 36b27b9c8024883857eba4328b1ef874603c4660 Author: Sahil Takiar Date: Fri Jun 1 12:25:01 2018 -0500 HIVE-19765: Add Parquet specific tests to BlobstoreCliDriver diff --git a/itests/hive-blobstore/src/test/queries/clientpositive/parquet_buckets.q b/itests/hive-blobstore/src/test/queries/clientpositive/parquet_buckets.q new file mode 100644 index 0000000000..c5a2cd3f66 --- /dev/null +++ b/itests/hive-blobstore/src/test/queries/clientpositive/parquet_buckets.q @@ -0,0 +1,31 @@ +-- Test simple interaction with partitioned bucketed table with paquet format in blobstore + +SET hive.exec.dynamic.partition=true; +SET hive.exec.reducers.max=10; +SET hive.exec.dynamic.partition.mode=nonstrict; + +DROP TABLE blobstore_source; +CREATE TABLE blobstore_source(a STRING, b STRING, c DOUBLE) +ROW FORMAT DELIMITED +FIELDS TERMINATED BY ' ' +COLLECTION ITEMS TERMINATED BY '\t' +LINES TERMINATED BY '\n' +LOCATION '${hiveconf:test.blobstore.path.unique}/parquet_buckets/blobstore_source/'; +LOAD DATA LOCAL INPATH '../../data/files/3col_data.txt' INTO TABLE blobstore_source; + +DROP TABLE parquet_buckets; +CREATE TABLE parquet_buckets (a STRING, value DOUBLE) +PARTITIONED BY (b STRING) +CLUSTERED BY (a) INTO 10 BUCKETS +STORED AS PARQUET +LOCATION '${hiveconf:test.blobstore.path.unique}/parquet_buckets/parquet_buckets'; + +INSERT OVERWRITE TABLE parquet_buckets +PARTITION (b) +SELECT a, c, b FROM blobstore_source; +SELECT * FROM parquet_buckets; + +INSERT INTO TABLE parquet_buckets +PARTITION (b) +SELECT a, c, b FROM blobstore_source; +SELECT * FROM parquet_buckets; \ No newline at end of file diff --git a/itests/hive-blobstore/src/test/queries/clientpositive/parquet_format_nonpart.q b/itests/hive-blobstore/src/test/queries/clientpositive/parquet_format_nonpart.q new file mode 100644 index 0000000000..a7827e9e8f --- /dev/null +++ b/itests/hive-blobstore/src/test/queries/clientpositive/parquet_format_nonpart.q @@ -0,0 +1,30 @@ +-- Test INSERT OVERWRITE and INSERT INTO on parquet table in blobstore + +DROP TABLE blobstore_source; +CREATE TABLE blobstore_source(a STRING, b STRING, c DOUBLE) +ROW FORMAT DELIMITED +FIELDS TERMINATED BY ' ' +COLLECTION ITEMS TERMINATED BY '\t' +LINES TERMINATED BY '\n' +LOCATION '${hiveconf:test.blobstore.path.unique}/parquet_format_nonpart/blobstore_source/'; +LOAD DATA LOCAL INPATH '../../data/files/3col_data.txt' INTO TABLE blobstore_source; + +DROP TABLE parquet_table; +CREATE EXTERNAL TABLE parquet_table (a INT, b STRING, value DOUBLE) STORED AS PARQUET +LOCATION '${hiveconf:test.blobstore.path.unique}/parquet_format_nonpart/parquet_table'; + +INSERT OVERWRITE TABLE parquet_table +SELECT * FROM blobstore_source; + +SELECT * FROM parquet_table; +SELECT a FROM parquet_table GROUP BY a; +SELECT b FROM parquet_table GROUP BY b; +SELECT value FROM parquet_table GROUP BY value; + +INSERT INTO TABLE parquet_table +SELECT * FROM blobstore_source; + +SELECT * FROM parquet_table; +SELECT a FROM parquet_table GROUP BY a; +SELECT b FROM parquet_table GROUP BY b; +SELECT value FROM parquet_table GROUP BY value; \ No newline at end of file diff --git a/itests/hive-blobstore/src/test/queries/clientpositive/parquet_format_part.q b/itests/hive-blobstore/src/test/queries/clientpositive/parquet_format_part.q new file mode 100644 index 0000000000..e9924fe749 --- /dev/null +++ b/itests/hive-blobstore/src/test/queries/clientpositive/parquet_format_part.q @@ -0,0 +1,67 @@ +-- Test INSERT INTO and INSERT OVERWRITE on partitioned parquet table in blobstore + +DROP TABLE src_events; +CREATE TABLE src_events +( + log_id BIGINT, + `time` BIGINT, + uid BIGINT, + user_id BIGINT, + type INT, + event_data STRING, + session_id STRING, + full_uid BIGINT, + run_date STRING, + game_id INT, + event_name STRING +) +ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' +LOCATION '${hiveconf:test.blobstore.path.unique}/parquet_format_part/src_events/'; +LOAD DATA LOCAL INPATH '../../data/files/events.txt' INTO TABLE src_events; + +DROP TABLE parquet_events; +CREATE TABLE parquet_events +( + log_id BIGINT, + `time` BIGINT, + uid BIGINT, + user_id BIGINT, + type INT, + event_data STRING, + session_id STRING, + full_uid BIGINT +) +PARTITIONED BY (run_date STRING, game_id INT, event_name STRING) +STORED AS PARQUET +LOCATION '${hiveconf:test.blobstore.path.unique}/parquet_format_part/parquet_events'; + +SET hive.exec.dynamic.partition=true; +SET hive.exec.dynamic.partition.mode=nonstrict; + +INSERT OVERWRITE TABLE parquet_events PARTITION (run_date, game_id, event_name) +SELECT * FROM src_events; +SHOW PARTITIONS parquet_events; +SELECT COUNT(*) FROM parquet_events; +SELECT COUNT(*) FROM parquet_events WHERE run_date=20120921; +SELECT COUNT(*) FROM parquet_events WHERE run_date=20121121; + +INSERT OVERWRITE TABLE parquet_events PARTITION (run_date=201211, game_id, event_name) +SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid,game_id,event_name FROM src_events +WHERE SUBSTR(run_date,1,6)='201211'; +SHOW PARTITIONS parquet_events; +SELECT COUNT(*) FROM parquet_events; + +INSERT INTO TABLE parquet_events PARTITION (run_date=201209, game_id=39, event_name) +SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid,event_name FROM src_events +WHERE SUBSTR(run_date,1,6)='201209' AND game_id=39; +SELECT COUNT(*) FROM parquet_events; + +INSERT INTO TABLE parquet_events PARTITION (run_date=201209, game_id=39, event_name='hq_change') +SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events +WHERE SUBSTR(run_date,1,6)='201209' AND game_id=39 AND event_name='hq_change'; +SELECT COUNT(*) FROM parquet_events; + +INSERT OVERWRITE TABLE parquet_events PARTITION (run_date=201209, game_id=39, event_name='hq_change') +SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events +WHERE SUBSTR(run_date,1,6)='201209' AND game_id=39 AND event_name='hq_change'; +SELECT COUNT(*) FROM parquet_events; \ No newline at end of file diff --git a/itests/hive-blobstore/src/test/queries/clientpositive/parquet_nonstd_partitions_loc.q b/itests/hive-blobstore/src/test/queries/clientpositive/parquet_nonstd_partitions_loc.q new file mode 100644 index 0000000000..2d20b99df3 --- /dev/null +++ b/itests/hive-blobstore/src/test/queries/clientpositive/parquet_nonstd_partitions_loc.q @@ -0,0 +1,100 @@ +-- Test table in parquet format with non-standard partition locations in blobstore + +DROP TABLE src_events; +CREATE TABLE src_events +( + log_id BIGINT, + `time` BIGINT, + uid BIGINT, + user_id BIGINT, + type INT, + event_data STRING, + session_id STRING, + full_uid BIGINT, + run_date STRING, + game_id INT, + event_name STRING +) +ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' +LOCATION '${hiveconf:test.blobstore.path.unique}/parquet_nonstd_partitions_loc/src_events/'; +LOAD DATA LOCAL INPATH '../../data/files/events.txt' INTO TABLE src_events; + +DROP TABLE parquet_events; +CREATE TABLE parquet_events +( + log_id BIGINT, + `time` BIGINT, + uid BIGINT, + user_id BIGINT, + type INT, + event_data STRING, + session_id STRING, + full_uid BIGINT +) +PARTITIONED BY (run_date STRING, game_id INT, event_name STRING) +STORED AS PARQUET +LOCATION '${hiveconf:test.blobstore.path.unique}/parquet_nonstd_partitions_loc/parquet_events/'; + +SET hive.exec.dynamic.partition=true; +SET hive.exec.dynamic.partition.mode=nonstrict; + +INSERT OVERWRITE TABLE parquet_events PARTITION (run_date, game_id, event_name) +SELECT * FROM src_events; +SHOW PARTITIONS parquet_events; +SELECT COUNT(*) FROM parquet_events; + +-- verify INSERT OVERWRITE and INSERT INTO nonstandard partition location +ALTER TABLE parquet_events ADD PARTITION (run_date=201211, game_id=39, event_name='hq_change') +LOCATION '${hiveconf:test.blobstore.path.unique}/parquet_nonstd_partitions_loc/parquet_nonstd_loc/ns-part-1/'; +INSERT OVERWRITE TABLE parquet_events PARTITION (run_date=201211, game_id=39, event_name='hq_change') +SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events +WHERE SUBSTR(run_date,1,6)='201211'; +SHOW PARTITIONS parquet_events; +SELECT COUNT(*) FROM parquet_events; +INSERT INTO TABLE parquet_events PARTITION (run_date=201211, game_id=39, event_name='hq_change') +SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events +WHERE SUBSTR(run_date,1,6)='201211'; +SHOW PARTITIONS parquet_events; +SELECT COUNT(*) FROM parquet_events; + +SET hive.merge.mapfiles=false; + +-- verify INSERT OVERWRITE and INSERT INTO nonstandard partition location with hive.merge.mapfiles false +ALTER TABLE parquet_events ADD PARTITION (run_date=201209, game_id=39, event_name='hq_change') +LOCATION '${hiveconf:test.blobstore.path.unique}/parquet_nonstd_partitions_loc/parquet_nonstd_loc/ns-part-2/'; +INSERT OVERWRITE TABLE parquet_events PARTITION (run_date=201209, game_id=39, event_name='hq_change') +SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events +WHERE SUBSTR(run_date,1,6)='201209'; +INSERT INTO TABLE parquet_events PARTITION (run_date=201209, game_id=39, event_name='hq_change') +SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events +WHERE SUBSTR(run_date,1,6)='201209'; +SHOW PARTITIONS parquet_events; +SELECT COUNT(*) FROM parquet_events; + +-- verify dynamic INSERT OVERWRITE over all partitions (standard and nonstandard locations) with hive.merge.mapfiles false +INSERT OVERWRITE TABLE parquet_events PARTITION (run_date, game_id, event_name) +SELECT * FROM src_events; +SHOW PARTITIONS parquet_events; +SELECT COUNT(*) FROM parquet_events; + +SET hive.merge.mapfiles=true; + +ALTER TABLE parquet_events ADD PARTITION (run_date=201207, game_id=39, event_name='hq_change') +LOCATION '${hiveconf:test.blobstore.path.unique}/parquet_nonstd_partitions_loc/parquet_nonstd_loc/ns-part-3/'; +INSERT INTO TABLE parquet_events PARTITION (run_date=201207, game_id=39, event_name='hq_change') +SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events +WHERE SUBSTR(run_date,1,6)='201209'; +SHOW PARTITIONS parquet_events; +SELECT COUNT(*) FROM parquet_events; + +-- verify dynamic INSERT OVERWRITE over all partitions (standard and nonstandard locations) with hive.merge.mapfiles true +INSERT OVERWRITE TABLE parquet_events PARTITION (run_date, game_id, event_name) +SELECT * FROM src_events; +SHOW PARTITIONS parquet_events; +SELECT COUNT(*) FROM parquet_events; + +ALTER TABLE parquet_events DROP PARTITION (run_date=201211, game_id=39, event_name='hq_change'); +ALTER TABLE parquet_events DROP PARTITION (run_date=201209, game_id=39, event_name='hq_change'); +ALTER TABLE parquet_events DROP PARTITION (run_date=201207, game_id=39, event_name='hq_change'); +SHOW PARTITIONS parquet_events; +SELECT COUNT(*) FROM parquet_events; \ No newline at end of file diff --git a/itests/hive-blobstore/src/test/queries/clientpositive/ptf_parquetfile.q b/itests/hive-blobstore/src/test/queries/clientpositive/ptf_parquetfile.q new file mode 100644 index 0000000000..fbf5ff302f --- /dev/null +++ b/itests/hive-blobstore/src/test/queries/clientpositive/ptf_parquetfile.q @@ -0,0 +1,32 @@ +-- Test several window functions on a table stored using PARQUET +DROP TABLE part_parquet; +CREATE TABLE part_parquet( + p_partkey int, + p_name string, + p_mfgr string, + p_brand string, + p_type string, + p_size int, + p_container string, + p_retailprice double, + p_comment string +) +STORED AS PARQUET +LOCATION '${hiveconf:test.blobstore.path.unique}/ptf_parquetfile/part_parquet'; + +LOAD DATA LOCAL INPATH '../../data/files/part.parquet' OVERWRITE INTO TABLE part_parquet; + +-- Test windowing PTFs with several partitions, using PARQUET storage +SELECT + p_mfgr, p_name, p_size, + RANK() OVER (PARTITION BY p_mfgr ORDER BY p_name) AS r, + DENSE_RANK() OVER (PARTITION BY p_mfgr ORDER BY p_name) AS dr, + SUM(p_retailprice) OVER ( + PARTITION BY p_mfgr + ORDER BY p_name ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW + ) AS s1 +FROM NOOP( + ON part_parquet + PARTITION BY p_mfgr + ORDER BY p_name +); diff --git a/itests/hive-blobstore/src/test/results/clientpositive/parquet_buckets.q.out b/itests/hive-blobstore/src/test/results/clientpositive/parquet_buckets.q.out new file mode 100644 index 0000000000..355958d5e1 --- /dev/null +++ b/itests/hive-blobstore/src/test/results/clientpositive/parquet_buckets.q.out @@ -0,0 +1,183 @@ +PREHOOK: query: DROP TABLE blobstore_source +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE blobstore_source +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE blobstore_source(a STRING, b STRING, c DOUBLE) +ROW FORMAT DELIMITED +FIELDS TERMINATED BY ' ' +COLLECTION ITEMS TERMINATED BY '\t' +LINES TERMINATED BY '\n' +#### A masked pattern was here #### +PREHOOK: type: CREATETABLE +PREHOOK: Input: ### test.blobstore.path ###/parquet_buckets/blobstore_source +PREHOOK: Output: database:default +PREHOOK: Output: default@blobstore_source +POSTHOOK: query: CREATE TABLE blobstore_source(a STRING, b STRING, c DOUBLE) +ROW FORMAT DELIMITED +FIELDS TERMINATED BY ' ' +COLLECTION ITEMS TERMINATED BY '\t' +LINES TERMINATED BY '\n' +#### A masked pattern was here #### +POSTHOOK: type: CREATETABLE +POSTHOOK: Input: ### test.blobstore.path ###/parquet_buckets/blobstore_source +POSTHOOK: Output: database:default +POSTHOOK: Output: default@blobstore_source +PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/3col_data.txt' INTO TABLE blobstore_source +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@blobstore_source +POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/3col_data.txt' INTO TABLE blobstore_source +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@blobstore_source +PREHOOK: query: DROP TABLE parquet_buckets +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE parquet_buckets +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE parquet_buckets (a STRING, value DOUBLE) +PARTITIONED BY (b STRING) +CLUSTERED BY (a) INTO 10 BUCKETS +STORED AS PARQUET +#### A masked pattern was here #### +PREHOOK: type: CREATETABLE +PREHOOK: Input: ### test.blobstore.path ###/parquet_buckets/parquet_buckets +PREHOOK: Output: database:default +PREHOOK: Output: default@parquet_buckets +POSTHOOK: query: CREATE TABLE parquet_buckets (a STRING, value DOUBLE) +PARTITIONED BY (b STRING) +CLUSTERED BY (a) INTO 10 BUCKETS +STORED AS PARQUET +#### A masked pattern was here #### +POSTHOOK: type: CREATETABLE +POSTHOOK: Input: ### test.blobstore.path ###/parquet_buckets/parquet_buckets +POSTHOOK: Output: database:default +POSTHOOK: Output: default@parquet_buckets +PREHOOK: query: INSERT OVERWRITE TABLE parquet_buckets +PARTITION (b) +SELECT a, c, b FROM blobstore_source +PREHOOK: type: QUERY +PREHOOK: Input: default@blobstore_source +PREHOOK: Output: default@parquet_buckets +POSTHOOK: query: INSERT OVERWRITE TABLE parquet_buckets +PARTITION (b) +SELECT a, c, b FROM blobstore_source +POSTHOOK: type: QUERY +POSTHOOK: Input: default@blobstore_source +POSTHOOK: Output: default@parquet_buckets@b=abc +POSTHOOK: Output: default@parquet_buckets@b=ajss +POSTHOOK: Output: default@parquet_buckets@b=data +POSTHOOK: Output: default@parquet_buckets@b=def +POSTHOOK: Output: default@parquet_buckets@b=djns +POSTHOOK: Output: default@parquet_buckets@b=ne +POSTHOOK: Output: default@parquet_buckets@b=random +POSTHOOK: Lineage: parquet_buckets PARTITION(b=abc).a SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:a, type:string, comment:null), ] +POSTHOOK: Lineage: parquet_buckets PARTITION(b=abc).value SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:c, type:double, comment:null), ] +POSTHOOK: Lineage: parquet_buckets PARTITION(b=ajss).a SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:a, type:string, comment:null), ] +POSTHOOK: Lineage: parquet_buckets PARTITION(b=ajss).value SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:c, type:double, comment:null), ] +POSTHOOK: Lineage: parquet_buckets PARTITION(b=data).a SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:a, type:string, comment:null), ] +POSTHOOK: Lineage: parquet_buckets PARTITION(b=data).value SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:c, type:double, comment:null), ] +POSTHOOK: Lineage: parquet_buckets PARTITION(b=def).a SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:a, type:string, comment:null), ] +POSTHOOK: Lineage: parquet_buckets PARTITION(b=def).value SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:c, type:double, comment:null), ] +POSTHOOK: Lineage: parquet_buckets PARTITION(b=djns).a SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:a, type:string, comment:null), ] +POSTHOOK: Lineage: parquet_buckets PARTITION(b=djns).value SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:c, type:double, comment:null), ] +POSTHOOK: Lineage: parquet_buckets PARTITION(b=ne).a SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:a, type:string, comment:null), ] +POSTHOOK: Lineage: parquet_buckets PARTITION(b=ne).value SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:c, type:double, comment:null), ] +POSTHOOK: Lineage: parquet_buckets PARTITION(b=random).a SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:a, type:string, comment:null), ] +POSTHOOK: Lineage: parquet_buckets PARTITION(b=random).value SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:c, type:double, comment:null), ] +PREHOOK: query: SELECT * FROM parquet_buckets +PREHOOK: type: QUERY +PREHOOK: Input: default@parquet_buckets +PREHOOK: Input: default@parquet_buckets@b=abc +PREHOOK: Input: default@parquet_buckets@b=ajss +PREHOOK: Input: default@parquet_buckets@b=data +PREHOOK: Input: default@parquet_buckets@b=def +PREHOOK: Input: default@parquet_buckets@b=djns +PREHOOK: Input: default@parquet_buckets@b=ne +PREHOOK: Input: default@parquet_buckets@b=random +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM parquet_buckets +POSTHOOK: type: QUERY +POSTHOOK: Input: default@parquet_buckets +POSTHOOK: Input: default@parquet_buckets@b=abc +POSTHOOK: Input: default@parquet_buckets@b=ajss +POSTHOOK: Input: default@parquet_buckets@b=data +POSTHOOK: Input: default@parquet_buckets@b=def +POSTHOOK: Input: default@parquet_buckets@b=djns +POSTHOOK: Input: default@parquet_buckets@b=ne +POSTHOOK: Input: default@parquet_buckets@b=random +#### A masked pattern was here #### +1 10.5 abc +3 90.23232 ajss +6 3.002 data +2 11.5 def +4 89.02002 djns +7 71.9084 ne +5 2.99 random +PREHOOK: query: INSERT INTO TABLE parquet_buckets +PARTITION (b) +SELECT a, c, b FROM blobstore_source +PREHOOK: type: QUERY +PREHOOK: Input: default@blobstore_source +PREHOOK: Output: default@parquet_buckets +POSTHOOK: query: INSERT INTO TABLE parquet_buckets +PARTITION (b) +SELECT a, c, b FROM blobstore_source +POSTHOOK: type: QUERY +POSTHOOK: Input: default@blobstore_source +POSTHOOK: Output: default@parquet_buckets@b=abc +POSTHOOK: Output: default@parquet_buckets@b=ajss +POSTHOOK: Output: default@parquet_buckets@b=data +POSTHOOK: Output: default@parquet_buckets@b=def +POSTHOOK: Output: default@parquet_buckets@b=djns +POSTHOOK: Output: default@parquet_buckets@b=ne +POSTHOOK: Output: default@parquet_buckets@b=random +POSTHOOK: Lineage: parquet_buckets PARTITION(b=abc).a SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:a, type:string, comment:null), ] +POSTHOOK: Lineage: parquet_buckets PARTITION(b=abc).value SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:c, type:double, comment:null), ] +POSTHOOK: Lineage: parquet_buckets PARTITION(b=ajss).a SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:a, type:string, comment:null), ] +POSTHOOK: Lineage: parquet_buckets PARTITION(b=ajss).value SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:c, type:double, comment:null), ] +POSTHOOK: Lineage: parquet_buckets PARTITION(b=data).a SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:a, type:string, comment:null), ] +POSTHOOK: Lineage: parquet_buckets PARTITION(b=data).value SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:c, type:double, comment:null), ] +POSTHOOK: Lineage: parquet_buckets PARTITION(b=def).a SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:a, type:string, comment:null), ] +POSTHOOK: Lineage: parquet_buckets PARTITION(b=def).value SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:c, type:double, comment:null), ] +POSTHOOK: Lineage: parquet_buckets PARTITION(b=djns).a SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:a, type:string, comment:null), ] +POSTHOOK: Lineage: parquet_buckets PARTITION(b=djns).value SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:c, type:double, comment:null), ] +POSTHOOK: Lineage: parquet_buckets PARTITION(b=ne).a SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:a, type:string, comment:null), ] +POSTHOOK: Lineage: parquet_buckets PARTITION(b=ne).value SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:c, type:double, comment:null), ] +POSTHOOK: Lineage: parquet_buckets PARTITION(b=random).a SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:a, type:string, comment:null), ] +POSTHOOK: Lineage: parquet_buckets PARTITION(b=random).value SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:c, type:double, comment:null), ] +PREHOOK: query: SELECT * FROM parquet_buckets +PREHOOK: type: QUERY +PREHOOK: Input: default@parquet_buckets +PREHOOK: Input: default@parquet_buckets@b=abc +PREHOOK: Input: default@parquet_buckets@b=ajss +PREHOOK: Input: default@parquet_buckets@b=data +PREHOOK: Input: default@parquet_buckets@b=def +PREHOOK: Input: default@parquet_buckets@b=djns +PREHOOK: Input: default@parquet_buckets@b=ne +PREHOOK: Input: default@parquet_buckets@b=random +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM parquet_buckets +POSTHOOK: type: QUERY +POSTHOOK: Input: default@parquet_buckets +POSTHOOK: Input: default@parquet_buckets@b=abc +POSTHOOK: Input: default@parquet_buckets@b=ajss +POSTHOOK: Input: default@parquet_buckets@b=data +POSTHOOK: Input: default@parquet_buckets@b=def +POSTHOOK: Input: default@parquet_buckets@b=djns +POSTHOOK: Input: default@parquet_buckets@b=ne +POSTHOOK: Input: default@parquet_buckets@b=random +#### A masked pattern was here #### +1 10.5 abc +1 10.5 abc +3 90.23232 ajss +3 90.23232 ajss +6 3.002 data +6 3.002 data +2 11.5 def +2 11.5 def +4 89.02002 djns +4 89.02002 djns +7 71.9084 ne +7 71.9084 ne +5 2.99 random +5 2.99 random diff --git a/itests/hive-blobstore/src/test/results/clientpositive/parquet_format_nonpart.q.out b/itests/hive-blobstore/src/test/results/clientpositive/parquet_format_nonpart.q.out new file mode 100644 index 0000000000..cba6b1a826 --- /dev/null +++ b/itests/hive-blobstore/src/test/results/clientpositive/parquet_format_nonpart.q.out @@ -0,0 +1,201 @@ +PREHOOK: query: DROP TABLE blobstore_source +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE blobstore_source +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE blobstore_source(a STRING, b STRING, c DOUBLE) +ROW FORMAT DELIMITED +FIELDS TERMINATED BY ' ' +COLLECTION ITEMS TERMINATED BY '\t' +LINES TERMINATED BY '\n' +#### A masked pattern was here #### +PREHOOK: type: CREATETABLE +PREHOOK: Input: ### test.blobstore.path ###/parquet_format_nonpart/blobstore_source +PREHOOK: Output: database:default +PREHOOK: Output: default@blobstore_source +POSTHOOK: query: CREATE TABLE blobstore_source(a STRING, b STRING, c DOUBLE) +ROW FORMAT DELIMITED +FIELDS TERMINATED BY ' ' +COLLECTION ITEMS TERMINATED BY '\t' +LINES TERMINATED BY '\n' +#### A masked pattern was here #### +POSTHOOK: type: CREATETABLE +POSTHOOK: Input: ### test.blobstore.path ###/parquet_format_nonpart/blobstore_source +POSTHOOK: Output: database:default +POSTHOOK: Output: default@blobstore_source +PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/3col_data.txt' INTO TABLE blobstore_source +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@blobstore_source +POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/3col_data.txt' INTO TABLE blobstore_source +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@blobstore_source +PREHOOK: query: DROP TABLE parquet_table +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE parquet_table +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE EXTERNAL TABLE parquet_table (a INT, b STRING, value DOUBLE) STORED AS PARQUET +#### A masked pattern was here #### +PREHOOK: type: CREATETABLE +PREHOOK: Input: ### test.blobstore.path ###/parquet_format_nonpart/parquet_table +PREHOOK: Output: database:default +PREHOOK: Output: default@parquet_table +POSTHOOK: query: CREATE EXTERNAL TABLE parquet_table (a INT, b STRING, value DOUBLE) STORED AS PARQUET +#### A masked pattern was here #### +POSTHOOK: type: CREATETABLE +POSTHOOK: Input: ### test.blobstore.path ###/parquet_format_nonpart/parquet_table +POSTHOOK: Output: database:default +POSTHOOK: Output: default@parquet_table +PREHOOK: query: INSERT OVERWRITE TABLE parquet_table +SELECT * FROM blobstore_source +PREHOOK: type: QUERY +PREHOOK: Input: default@blobstore_source +PREHOOK: Output: default@parquet_table +POSTHOOK: query: INSERT OVERWRITE TABLE parquet_table +SELECT * FROM blobstore_source +POSTHOOK: type: QUERY +POSTHOOK: Input: default@blobstore_source +POSTHOOK: Output: default@parquet_table +POSTHOOK: Lineage: parquet_table.a EXPRESSION [(blobstore_source)blobstore_source.FieldSchema(name:a, type:string, comment:null), ] +POSTHOOK: Lineage: parquet_table.b SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:b, type:string, comment:null), ] +POSTHOOK: Lineage: parquet_table.value SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:c, type:double, comment:null), ] +PREHOOK: query: SELECT * FROM parquet_table +PREHOOK: type: QUERY +PREHOOK: Input: default@parquet_table +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM parquet_table +POSTHOOK: type: QUERY +POSTHOOK: Input: default@parquet_table +#### A masked pattern was here #### +1 abc 10.5 +2 def 11.5 +3 ajss 90.23232 +4 djns 89.02002 +5 random 2.99 +6 data 3.002 +7 ne 71.9084 +PREHOOK: query: SELECT a FROM parquet_table GROUP BY a +PREHOOK: type: QUERY +PREHOOK: Input: default@parquet_table +#### A masked pattern was here #### +POSTHOOK: query: SELECT a FROM parquet_table GROUP BY a +POSTHOOK: type: QUERY +POSTHOOK: Input: default@parquet_table +#### A masked pattern was here #### +1 +2 +3 +4 +5 +6 +7 +PREHOOK: query: SELECT b FROM parquet_table GROUP BY b +PREHOOK: type: QUERY +PREHOOK: Input: default@parquet_table +#### A masked pattern was here #### +POSTHOOK: query: SELECT b FROM parquet_table GROUP BY b +POSTHOOK: type: QUERY +POSTHOOK: Input: default@parquet_table +#### A masked pattern was here #### +abc +ajss +data +def +djns +ne +random +PREHOOK: query: SELECT value FROM parquet_table GROUP BY value +PREHOOK: type: QUERY +PREHOOK: Input: default@parquet_table +#### A masked pattern was here #### +POSTHOOK: query: SELECT value FROM parquet_table GROUP BY value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@parquet_table +#### A masked pattern was here #### +2.99 +3.002 +10.5 +11.5 +71.9084 +89.02002 +90.23232 +PREHOOK: query: INSERT INTO TABLE parquet_table +SELECT * FROM blobstore_source +PREHOOK: type: QUERY +PREHOOK: Input: default@blobstore_source +PREHOOK: Output: default@parquet_table +POSTHOOK: query: INSERT INTO TABLE parquet_table +SELECT * FROM blobstore_source +POSTHOOK: type: QUERY +POSTHOOK: Input: default@blobstore_source +POSTHOOK: Output: default@parquet_table +POSTHOOK: Lineage: parquet_table.a EXPRESSION [(blobstore_source)blobstore_source.FieldSchema(name:a, type:string, comment:null), ] +POSTHOOK: Lineage: parquet_table.b SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:b, type:string, comment:null), ] +POSTHOOK: Lineage: parquet_table.value SIMPLE [(blobstore_source)blobstore_source.FieldSchema(name:c, type:double, comment:null), ] +PREHOOK: query: SELECT * FROM parquet_table +PREHOOK: type: QUERY +PREHOOK: Input: default@parquet_table +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM parquet_table +POSTHOOK: type: QUERY +POSTHOOK: Input: default@parquet_table +#### A masked pattern was here #### +1 abc 10.5 +2 def 11.5 +3 ajss 90.23232 +4 djns 89.02002 +5 random 2.99 +6 data 3.002 +7 ne 71.9084 +1 abc 10.5 +2 def 11.5 +3 ajss 90.23232 +4 djns 89.02002 +5 random 2.99 +6 data 3.002 +7 ne 71.9084 +PREHOOK: query: SELECT a FROM parquet_table GROUP BY a +PREHOOK: type: QUERY +PREHOOK: Input: default@parquet_table +#### A masked pattern was here #### +POSTHOOK: query: SELECT a FROM parquet_table GROUP BY a +POSTHOOK: type: QUERY +POSTHOOK: Input: default@parquet_table +#### A masked pattern was here #### +1 +2 +3 +4 +5 +6 +7 +PREHOOK: query: SELECT b FROM parquet_table GROUP BY b +PREHOOK: type: QUERY +PREHOOK: Input: default@parquet_table +#### A masked pattern was here #### +POSTHOOK: query: SELECT b FROM parquet_table GROUP BY b +POSTHOOK: type: QUERY +POSTHOOK: Input: default@parquet_table +#### A masked pattern was here #### +abc +ajss +data +def +djns +ne +random +PREHOOK: query: SELECT value FROM parquet_table GROUP BY value +PREHOOK: type: QUERY +PREHOOK: Input: default@parquet_table +#### A masked pattern was here #### +POSTHOOK: query: SELECT value FROM parquet_table GROUP BY value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@parquet_table +#### A masked pattern was here #### +2.99 +3.002 +10.5 +11.5 +71.9084 +89.02002 +90.23232 diff --git a/itests/hive-blobstore/src/test/results/clientpositive/parquet_format_part.q.out b/itests/hive-blobstore/src/test/results/clientpositive/parquet_format_part.q.out new file mode 100644 index 0000000000..0931e3d557 --- /dev/null +++ b/itests/hive-blobstore/src/test/results/clientpositive/parquet_format_part.q.out @@ -0,0 +1,290 @@ +PREHOOK: query: DROP TABLE src_events +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE src_events +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE src_events +( + log_id BIGINT, + `time` BIGINT, + uid BIGINT, + user_id BIGINT, + type INT, + event_data STRING, + session_id STRING, + full_uid BIGINT, + run_date STRING, + game_id INT, + event_name STRING +) +ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' +#### A masked pattern was here #### +PREHOOK: type: CREATETABLE +PREHOOK: Input: ### test.blobstore.path ###/parquet_format_part/src_events +PREHOOK: Output: database:default +PREHOOK: Output: default@src_events +POSTHOOK: query: CREATE TABLE src_events +( + log_id BIGINT, + `time` BIGINT, + uid BIGINT, + user_id BIGINT, + type INT, + event_data STRING, + session_id STRING, + full_uid BIGINT, + run_date STRING, + game_id INT, + event_name STRING +) +ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' +#### A masked pattern was here #### +POSTHOOK: type: CREATETABLE +POSTHOOK: Input: ### test.blobstore.path ###/parquet_format_part/src_events +POSTHOOK: Output: database:default +POSTHOOK: Output: default@src_events +PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/events.txt' INTO TABLE src_events +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@src_events +POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/events.txt' INTO TABLE src_events +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@src_events +PREHOOK: query: DROP TABLE parquet_events +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE parquet_events +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE parquet_events +( + log_id BIGINT, + `time` BIGINT, + uid BIGINT, + user_id BIGINT, + type INT, + event_data STRING, + session_id STRING, + full_uid BIGINT +) +PARTITIONED BY (run_date STRING, game_id INT, event_name STRING) +STORED AS PARQUET +#### A masked pattern was here #### +PREHOOK: type: CREATETABLE +PREHOOK: Input: ### test.blobstore.path ###/parquet_format_part/parquet_events +PREHOOK: Output: database:default +PREHOOK: Output: default@parquet_events +POSTHOOK: query: CREATE TABLE parquet_events +( + log_id BIGINT, + `time` BIGINT, + uid BIGINT, + user_id BIGINT, + type INT, + event_data STRING, + session_id STRING, + full_uid BIGINT +) +PARTITIONED BY (run_date STRING, game_id INT, event_name STRING) +STORED AS PARQUET +#### A masked pattern was here #### +POSTHOOK: type: CREATETABLE +POSTHOOK: Input: ### test.blobstore.path ###/parquet_format_part/parquet_events +POSTHOOK: Output: database:default +POSTHOOK: Output: default@parquet_events +PREHOOK: query: INSERT OVERWRITE TABLE parquet_events PARTITION (run_date, game_id, event_name) +SELECT * FROM src_events +PREHOOK: type: QUERY +PREHOOK: Input: default@src_events +PREHOOK: Output: default@parquet_events +POSTHOOK: query: INSERT OVERWRITE TABLE parquet_events PARTITION (run_date, game_id, event_name) +SELECT * FROM src_events +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src_events +POSTHOOK: Output: default@parquet_events@run_date=20120921/game_id=39/event_name=hq_change +POSTHOOK: Output: default@parquet_events@run_date=20121021/game_id=39/event_name=hq_change +POSTHOOK: Output: default@parquet_events@run_date=20121121/game_id=39/event_name=hq_change +POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).event_data SIMPLE [(src_events)src_events.FieldSchema(name:event_data, type:string, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).full_uid SIMPLE [(src_events)src_events.FieldSchema(name:full_uid, type:bigint, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).log_id SIMPLE [(src_events)src_events.FieldSchema(name:log_id, type:bigint, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).session_id SIMPLE [(src_events)src_events.FieldSchema(name:session_id, type:string, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).time SIMPLE [(src_events)src_events.FieldSchema(name:time, type:bigint, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).type SIMPLE [(src_events)src_events.FieldSchema(name:type, type:int, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).uid SIMPLE [(src_events)src_events.FieldSchema(name:uid, type:bigint, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).user_id SIMPLE [(src_events)src_events.FieldSchema(name:user_id, type:bigint, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).event_data SIMPLE [(src_events)src_events.FieldSchema(name:event_data, type:string, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).full_uid SIMPLE [(src_events)src_events.FieldSchema(name:full_uid, type:bigint, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).log_id SIMPLE [(src_events)src_events.FieldSchema(name:log_id, type:bigint, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).session_id SIMPLE [(src_events)src_events.FieldSchema(name:session_id, type:string, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).time SIMPLE [(src_events)src_events.FieldSchema(name:time, type:bigint, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).type SIMPLE [(src_events)src_events.FieldSchema(name:type, type:int, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).uid SIMPLE [(src_events)src_events.FieldSchema(name:uid, type:bigint, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).user_id SIMPLE [(src_events)src_events.FieldSchema(name:user_id, type:bigint, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).event_data SIMPLE [(src_events)src_events.FieldSchema(name:event_data, type:string, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).full_uid SIMPLE [(src_events)src_events.FieldSchema(name:full_uid, type:bigint, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).log_id SIMPLE [(src_events)src_events.FieldSchema(name:log_id, type:bigint, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).session_id SIMPLE [(src_events)src_events.FieldSchema(name:session_id, type:string, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).time SIMPLE [(src_events)src_events.FieldSchema(name:time, type:bigint, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).type SIMPLE [(src_events)src_events.FieldSchema(name:type, type:int, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).uid SIMPLE [(src_events)src_events.FieldSchema(name:uid, type:bigint, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).user_id SIMPLE [(src_events)src_events.FieldSchema(name:user_id, type:bigint, comment:null), ] +PREHOOK: query: SHOW PARTITIONS parquet_events +PREHOOK: type: SHOWPARTITIONS +PREHOOK: Input: default@parquet_events +POSTHOOK: query: SHOW PARTITIONS parquet_events +POSTHOOK: type: SHOWPARTITIONS +POSTHOOK: Input: default@parquet_events +run_date=20120921/game_id=39/event_name=hq_change +run_date=20121021/game_id=39/event_name=hq_change +run_date=20121121/game_id=39/event_name=hq_change +PREHOOK: query: SELECT COUNT(*) FROM parquet_events +PREHOOK: type: QUERY +PREHOOK: Input: default@parquet_events +#### A masked pattern was here #### +POSTHOOK: query: SELECT COUNT(*) FROM parquet_events +POSTHOOK: type: QUERY +POSTHOOK: Input: default@parquet_events +#### A masked pattern was here #### +200 +PREHOOK: query: SELECT COUNT(*) FROM parquet_events WHERE run_date=20120921 +PREHOOK: type: QUERY +PREHOOK: Input: default@parquet_events +#### A masked pattern was here #### +POSTHOOK: query: SELECT COUNT(*) FROM parquet_events WHERE run_date=20120921 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@parquet_events +#### A masked pattern was here #### +50 +PREHOOK: query: SELECT COUNT(*) FROM parquet_events WHERE run_date=20121121 +PREHOOK: type: QUERY +PREHOOK: Input: default@parquet_events +#### A masked pattern was here #### +POSTHOOK: query: SELECT COUNT(*) FROM parquet_events WHERE run_date=20121121 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@parquet_events +#### A masked pattern was here #### +100 +PREHOOK: query: INSERT OVERWRITE TABLE parquet_events PARTITION (run_date=201211, game_id, event_name) +SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid,game_id,event_name FROM src_events +WHERE SUBSTR(run_date,1,6)='201211' +PREHOOK: type: QUERY +PREHOOK: Input: default@src_events +PREHOOK: Output: default@parquet_events@run_date=201211 +POSTHOOK: query: INSERT OVERWRITE TABLE parquet_events PARTITION (run_date=201211, game_id, event_name) +SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid,game_id,event_name FROM src_events +WHERE SUBSTR(run_date,1,6)='201211' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src_events +POSTHOOK: Output: default@parquet_events@run_date=201211/game_id=39/event_name=hq_change +POSTHOOK: Lineage: parquet_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).event_data SIMPLE [(src_events)src_events.FieldSchema(name:event_data, type:string, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).full_uid SIMPLE [(src_events)src_events.FieldSchema(name:full_uid, type:bigint, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).log_id SIMPLE [(src_events)src_events.FieldSchema(name:log_id, type:bigint, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).session_id SIMPLE [(src_events)src_events.FieldSchema(name:session_id, type:string, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).time SIMPLE [(src_events)src_events.FieldSchema(name:time, type:bigint, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).type SIMPLE [(src_events)src_events.FieldSchema(name:type, type:int, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).uid SIMPLE [(src_events)src_events.FieldSchema(name:uid, type:bigint, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).user_id SIMPLE [(src_events)src_events.FieldSchema(name:user_id, type:bigint, comment:null), ] +PREHOOK: query: SHOW PARTITIONS parquet_events +PREHOOK: type: SHOWPARTITIONS +PREHOOK: Input: default@parquet_events +POSTHOOK: query: SHOW PARTITIONS parquet_events +POSTHOOK: type: SHOWPARTITIONS +POSTHOOK: Input: default@parquet_events +run_date=20120921/game_id=39/event_name=hq_change +run_date=20121021/game_id=39/event_name=hq_change +run_date=201211/game_id=39/event_name=hq_change +run_date=20121121/game_id=39/event_name=hq_change +PREHOOK: query: SELECT COUNT(*) FROM parquet_events +PREHOOK: type: QUERY +PREHOOK: Input: default@parquet_events +#### A masked pattern was here #### +POSTHOOK: query: SELECT COUNT(*) FROM parquet_events +POSTHOOK: type: QUERY +POSTHOOK: Input: default@parquet_events +#### A masked pattern was here #### +300 +PREHOOK: query: INSERT INTO TABLE parquet_events PARTITION (run_date=201209, game_id=39, event_name) +SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid,event_name FROM src_events +WHERE SUBSTR(run_date,1,6)='201209' AND game_id=39 +PREHOOK: type: QUERY +PREHOOK: Input: default@src_events +PREHOOK: Output: default@parquet_events@run_date=201209/game_id=39 +POSTHOOK: query: INSERT INTO TABLE parquet_events PARTITION (run_date=201209, game_id=39, event_name) +SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid,event_name FROM src_events +WHERE SUBSTR(run_date,1,6)='201209' AND game_id=39 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src_events +POSTHOOK: Output: default@parquet_events@run_date=201209/game_id=39/event_name=hq_change +POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).event_data SIMPLE [(src_events)src_events.FieldSchema(name:event_data, type:string, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).full_uid SIMPLE [(src_events)src_events.FieldSchema(name:full_uid, type:bigint, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).log_id SIMPLE [(src_events)src_events.FieldSchema(name:log_id, type:bigint, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).session_id SIMPLE [(src_events)src_events.FieldSchema(name:session_id, type:string, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).time SIMPLE [(src_events)src_events.FieldSchema(name:time, type:bigint, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).type SIMPLE [(src_events)src_events.FieldSchema(name:type, type:int, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).uid SIMPLE [(src_events)src_events.FieldSchema(name:uid, type:bigint, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).user_id SIMPLE [(src_events)src_events.FieldSchema(name:user_id, type:bigint, comment:null), ] +PREHOOK: query: SELECT COUNT(*) FROM parquet_events +PREHOOK: type: QUERY +PREHOOK: Input: default@parquet_events +#### A masked pattern was here #### +POSTHOOK: query: SELECT COUNT(*) FROM parquet_events +POSTHOOK: type: QUERY +POSTHOOK: Input: default@parquet_events +#### A masked pattern was here #### +350 +PREHOOK: query: INSERT INTO TABLE parquet_events PARTITION (run_date=201209, game_id=39, event_name='hq_change') +SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events +WHERE SUBSTR(run_date,1,6)='201209' AND game_id=39 AND event_name='hq_change' +PREHOOK: type: QUERY +PREHOOK: Input: default@src_events +PREHOOK: Output: default@parquet_events@run_date=201209/game_id=39/event_name=hq_change +POSTHOOK: query: INSERT INTO TABLE parquet_events PARTITION (run_date=201209, game_id=39, event_name='hq_change') +SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events +WHERE SUBSTR(run_date,1,6)='201209' AND game_id=39 AND event_name='hq_change' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src_events +POSTHOOK: Output: default@parquet_events@run_date=201209/game_id=39/event_name=hq_change +POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).event_data SIMPLE [(src_events)src_events.FieldSchema(name:event_data, type:string, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).full_uid SIMPLE [(src_events)src_events.FieldSchema(name:full_uid, type:bigint, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).log_id SIMPLE [(src_events)src_events.FieldSchema(name:log_id, type:bigint, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).session_id SIMPLE [(src_events)src_events.FieldSchema(name:session_id, type:string, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).time SIMPLE [(src_events)src_events.FieldSchema(name:time, type:bigint, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).type SIMPLE [(src_events)src_events.FieldSchema(name:type, type:int, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).uid SIMPLE [(src_events)src_events.FieldSchema(name:uid, type:bigint, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).user_id SIMPLE [(src_events)src_events.FieldSchema(name:user_id, type:bigint, comment:null), ] +PREHOOK: query: SELECT COUNT(*) FROM parquet_events +PREHOOK: type: QUERY +PREHOOK: Input: default@parquet_events +#### A masked pattern was here #### +POSTHOOK: query: SELECT COUNT(*) FROM parquet_events +POSTHOOK: type: QUERY +POSTHOOK: Input: default@parquet_events +#### A masked pattern was here #### +400 +PREHOOK: query: INSERT OVERWRITE TABLE parquet_events PARTITION (run_date=201209, game_id=39, event_name='hq_change') +SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events +WHERE SUBSTR(run_date,1,6)='201209' AND game_id=39 AND event_name='hq_change' +PREHOOK: type: QUERY +PREHOOK: Input: default@src_events +PREHOOK: Output: default@parquet_events@run_date=201209/game_id=39/event_name=hq_change +POSTHOOK: query: INSERT OVERWRITE TABLE parquet_events PARTITION (run_date=201209, game_id=39, event_name='hq_change') +SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events +WHERE SUBSTR(run_date,1,6)='201209' AND game_id=39 AND event_name='hq_change' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src_events +POSTHOOK: Output: default@parquet_events@run_date=201209/game_id=39/event_name=hq_change +POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).event_data SIMPLE [(src_events)src_events.FieldSchema(name:event_data, type:string, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).full_uid SIMPLE [(src_events)src_events.FieldSchema(name:full_uid, type:bigint, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).log_id SIMPLE [(src_events)src_events.FieldSchema(name:log_id, type:bigint, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).session_id SIMPLE [(src_events)src_events.FieldSchema(name:session_id, type:string, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).time SIMPLE [(src_events)src_events.FieldSchema(name:time, type:bigint, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).type SIMPLE [(src_events)src_events.FieldSchema(name:type, type:int, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).uid SIMPLE [(src_events)src_events.FieldSchema(name:uid, type:bigint, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).user_id SIMPLE [(src_events)src_events.FieldSchema(name:user_id, type:bigint, comment:null), ] +PREHOOK: query: SELECT COUNT(*) FROM parquet_events +PREHOOK: type: QUERY +PREHOOK: Input: default@parquet_events +#### A masked pattern was here #### +POSTHOOK: query: SELECT COUNT(*) FROM parquet_events +POSTHOOK: type: QUERY +POSTHOOK: Input: default@parquet_events +#### A masked pattern was here #### +350 diff --git a/itests/hive-blobstore/src/test/results/clientpositive/parquet_nonstd_partitions_loc.q.out b/itests/hive-blobstore/src/test/results/clientpositive/parquet_nonstd_partitions_loc.q.out new file mode 100644 index 0000000000..15ae3d91a7 --- /dev/null +++ b/itests/hive-blobstore/src/test/results/clientpositive/parquet_nonstd_partitions_loc.q.out @@ -0,0 +1,537 @@ +PREHOOK: query: DROP TABLE src_events +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE src_events +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE src_events +( + log_id BIGINT, + `time` BIGINT, + uid BIGINT, + user_id BIGINT, + type INT, + event_data STRING, + session_id STRING, + full_uid BIGINT, + run_date STRING, + game_id INT, + event_name STRING +) +ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' +#### A masked pattern was here #### +PREHOOK: type: CREATETABLE +PREHOOK: Input: ### test.blobstore.path ###/parquet_nonstd_partitions_loc/src_events +PREHOOK: Output: database:default +PREHOOK: Output: default@src_events +POSTHOOK: query: CREATE TABLE src_events +( + log_id BIGINT, + `time` BIGINT, + uid BIGINT, + user_id BIGINT, + type INT, + event_data STRING, + session_id STRING, + full_uid BIGINT, + run_date STRING, + game_id INT, + event_name STRING +) +ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' +#### A masked pattern was here #### +POSTHOOK: type: CREATETABLE +POSTHOOK: Input: ### test.blobstore.path ###/parquet_nonstd_partitions_loc/src_events +POSTHOOK: Output: database:default +POSTHOOK: Output: default@src_events +PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/events.txt' INTO TABLE src_events +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@src_events +POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/events.txt' INTO TABLE src_events +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@src_events +PREHOOK: query: DROP TABLE parquet_events +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE parquet_events +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE parquet_events +( + log_id BIGINT, + `time` BIGINT, + uid BIGINT, + user_id BIGINT, + type INT, + event_data STRING, + session_id STRING, + full_uid BIGINT +) +PARTITIONED BY (run_date STRING, game_id INT, event_name STRING) +STORED AS PARQUET +#### A masked pattern was here #### +PREHOOK: type: CREATETABLE +PREHOOK: Input: ### test.blobstore.path ###/parquet_nonstd_partitions_loc/parquet_events +PREHOOK: Output: database:default +PREHOOK: Output: default@parquet_events +POSTHOOK: query: CREATE TABLE parquet_events +( + log_id BIGINT, + `time` BIGINT, + uid BIGINT, + user_id BIGINT, + type INT, + event_data STRING, + session_id STRING, + full_uid BIGINT +) +PARTITIONED BY (run_date STRING, game_id INT, event_name STRING) +STORED AS PARQUET +#### A masked pattern was here #### +POSTHOOK: type: CREATETABLE +POSTHOOK: Input: ### test.blobstore.path ###/parquet_nonstd_partitions_loc/parquet_events +POSTHOOK: Output: database:default +POSTHOOK: Output: default@parquet_events +PREHOOK: query: INSERT OVERWRITE TABLE parquet_events PARTITION (run_date, game_id, event_name) +SELECT * FROM src_events +PREHOOK: type: QUERY +PREHOOK: Input: default@src_events +PREHOOK: Output: default@parquet_events +POSTHOOK: query: INSERT OVERWRITE TABLE parquet_events PARTITION (run_date, game_id, event_name) +SELECT * FROM src_events +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src_events +POSTHOOK: Output: default@parquet_events@run_date=20120921/game_id=39/event_name=hq_change +POSTHOOK: Output: default@parquet_events@run_date=20121021/game_id=39/event_name=hq_change +POSTHOOK: Output: default@parquet_events@run_date=20121121/game_id=39/event_name=hq_change +POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).event_data SIMPLE [(src_events)src_events.FieldSchema(name:event_data, type:string, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).full_uid SIMPLE [(src_events)src_events.FieldSchema(name:full_uid, type:bigint, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).log_id SIMPLE [(src_events)src_events.FieldSchema(name:log_id, type:bigint, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).session_id SIMPLE [(src_events)src_events.FieldSchema(name:session_id, type:string, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).time SIMPLE [(src_events)src_events.FieldSchema(name:time, type:bigint, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).type SIMPLE [(src_events)src_events.FieldSchema(name:type, type:int, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).uid SIMPLE [(src_events)src_events.FieldSchema(name:uid, type:bigint, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).user_id SIMPLE [(src_events)src_events.FieldSchema(name:user_id, type:bigint, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).event_data SIMPLE [(src_events)src_events.FieldSchema(name:event_data, type:string, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).full_uid SIMPLE [(src_events)src_events.FieldSchema(name:full_uid, type:bigint, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).log_id SIMPLE [(src_events)src_events.FieldSchema(name:log_id, type:bigint, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).session_id SIMPLE [(src_events)src_events.FieldSchema(name:session_id, type:string, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).time SIMPLE [(src_events)src_events.FieldSchema(name:time, type:bigint, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).type SIMPLE [(src_events)src_events.FieldSchema(name:type, type:int, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).uid SIMPLE [(src_events)src_events.FieldSchema(name:uid, type:bigint, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).user_id SIMPLE [(src_events)src_events.FieldSchema(name:user_id, type:bigint, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).event_data SIMPLE [(src_events)src_events.FieldSchema(name:event_data, type:string, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).full_uid SIMPLE [(src_events)src_events.FieldSchema(name:full_uid, type:bigint, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).log_id SIMPLE [(src_events)src_events.FieldSchema(name:log_id, type:bigint, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).session_id SIMPLE [(src_events)src_events.FieldSchema(name:session_id, type:string, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).time SIMPLE [(src_events)src_events.FieldSchema(name:time, type:bigint, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).type SIMPLE [(src_events)src_events.FieldSchema(name:type, type:int, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).uid SIMPLE [(src_events)src_events.FieldSchema(name:uid, type:bigint, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).user_id SIMPLE [(src_events)src_events.FieldSchema(name:user_id, type:bigint, comment:null), ] +PREHOOK: query: SHOW PARTITIONS parquet_events +PREHOOK: type: SHOWPARTITIONS +PREHOOK: Input: default@parquet_events +POSTHOOK: query: SHOW PARTITIONS parquet_events +POSTHOOK: type: SHOWPARTITIONS +POSTHOOK: Input: default@parquet_events +run_date=20120921/game_id=39/event_name=hq_change +run_date=20121021/game_id=39/event_name=hq_change +run_date=20121121/game_id=39/event_name=hq_change +PREHOOK: query: SELECT COUNT(*) FROM parquet_events +PREHOOK: type: QUERY +PREHOOK: Input: default@parquet_events +#### A masked pattern was here #### +POSTHOOK: query: SELECT COUNT(*) FROM parquet_events +POSTHOOK: type: QUERY +POSTHOOK: Input: default@parquet_events +#### A masked pattern was here #### +200 +PREHOOK: query: ALTER TABLE parquet_events ADD PARTITION (run_date=201211, game_id=39, event_name='hq_change') +#### A masked pattern was here #### +PREHOOK: type: ALTERTABLE_ADDPARTS +PREHOOK: Input: ### test.blobstore.path ###/parquet_nonstd_partitions_loc/parquet_nonstd_loc/ns-part-1 +PREHOOK: Output: default@parquet_events +POSTHOOK: query: ALTER TABLE parquet_events ADD PARTITION (run_date=201211, game_id=39, event_name='hq_change') +#### A masked pattern was here #### +POSTHOOK: type: ALTERTABLE_ADDPARTS +POSTHOOK: Input: ### test.blobstore.path ###/parquet_nonstd_partitions_loc/parquet_nonstd_loc/ns-part-1 +POSTHOOK: Output: default@parquet_events +POSTHOOK: Output: default@parquet_events@run_date=201211/game_id=39/event_name=hq_change +PREHOOK: query: INSERT OVERWRITE TABLE parquet_events PARTITION (run_date=201211, game_id=39, event_name='hq_change') +SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events +WHERE SUBSTR(run_date,1,6)='201211' +PREHOOK: type: QUERY +PREHOOK: Input: default@src_events +PREHOOK: Output: default@parquet_events@run_date=201211/game_id=39/event_name=hq_change +POSTHOOK: query: INSERT OVERWRITE TABLE parquet_events PARTITION (run_date=201211, game_id=39, event_name='hq_change') +SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events +WHERE SUBSTR(run_date,1,6)='201211' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src_events +POSTHOOK: Output: default@parquet_events@run_date=201211/game_id=39/event_name=hq_change +POSTHOOK: Lineage: parquet_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).event_data SIMPLE [(src_events)src_events.FieldSchema(name:event_data, type:string, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).full_uid SIMPLE [(src_events)src_events.FieldSchema(name:full_uid, type:bigint, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).log_id SIMPLE [(src_events)src_events.FieldSchema(name:log_id, type:bigint, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).session_id SIMPLE [(src_events)src_events.FieldSchema(name:session_id, type:string, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).time SIMPLE [(src_events)src_events.FieldSchema(name:time, type:bigint, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).type SIMPLE [(src_events)src_events.FieldSchema(name:type, type:int, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).uid SIMPLE [(src_events)src_events.FieldSchema(name:uid, type:bigint, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).user_id SIMPLE [(src_events)src_events.FieldSchema(name:user_id, type:bigint, comment:null), ] +PREHOOK: query: SHOW PARTITIONS parquet_events +PREHOOK: type: SHOWPARTITIONS +PREHOOK: Input: default@parquet_events +POSTHOOK: query: SHOW PARTITIONS parquet_events +POSTHOOK: type: SHOWPARTITIONS +POSTHOOK: Input: default@parquet_events +run_date=20120921/game_id=39/event_name=hq_change +run_date=20121021/game_id=39/event_name=hq_change +run_date=201211/game_id=39/event_name=hq_change +run_date=20121121/game_id=39/event_name=hq_change +PREHOOK: query: SELECT COUNT(*) FROM parquet_events +PREHOOK: type: QUERY +PREHOOK: Input: default@parquet_events +#### A masked pattern was here #### +POSTHOOK: query: SELECT COUNT(*) FROM parquet_events +POSTHOOK: type: QUERY +POSTHOOK: Input: default@parquet_events +#### A masked pattern was here #### +300 +PREHOOK: query: INSERT INTO TABLE parquet_events PARTITION (run_date=201211, game_id=39, event_name='hq_change') +SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events +WHERE SUBSTR(run_date,1,6)='201211' +PREHOOK: type: QUERY +PREHOOK: Input: default@src_events +PREHOOK: Output: default@parquet_events@run_date=201211/game_id=39/event_name=hq_change +POSTHOOK: query: INSERT INTO TABLE parquet_events PARTITION (run_date=201211, game_id=39, event_name='hq_change') +SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events +WHERE SUBSTR(run_date,1,6)='201211' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src_events +POSTHOOK: Output: default@parquet_events@run_date=201211/game_id=39/event_name=hq_change +POSTHOOK: Lineage: parquet_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).event_data SIMPLE [(src_events)src_events.FieldSchema(name:event_data, type:string, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).full_uid SIMPLE [(src_events)src_events.FieldSchema(name:full_uid, type:bigint, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).log_id SIMPLE [(src_events)src_events.FieldSchema(name:log_id, type:bigint, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).session_id SIMPLE [(src_events)src_events.FieldSchema(name:session_id, type:string, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).time SIMPLE [(src_events)src_events.FieldSchema(name:time, type:bigint, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).type SIMPLE [(src_events)src_events.FieldSchema(name:type, type:int, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).uid SIMPLE [(src_events)src_events.FieldSchema(name:uid, type:bigint, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=201211,game_id=39,event_name=hq_change).user_id SIMPLE [(src_events)src_events.FieldSchema(name:user_id, type:bigint, comment:null), ] +PREHOOK: query: SHOW PARTITIONS parquet_events +PREHOOK: type: SHOWPARTITIONS +PREHOOK: Input: default@parquet_events +POSTHOOK: query: SHOW PARTITIONS parquet_events +POSTHOOK: type: SHOWPARTITIONS +POSTHOOK: Input: default@parquet_events +run_date=20120921/game_id=39/event_name=hq_change +run_date=20121021/game_id=39/event_name=hq_change +run_date=201211/game_id=39/event_name=hq_change +run_date=20121121/game_id=39/event_name=hq_change +PREHOOK: query: SELECT COUNT(*) FROM parquet_events +PREHOOK: type: QUERY +PREHOOK: Input: default@parquet_events +#### A masked pattern was here #### +POSTHOOK: query: SELECT COUNT(*) FROM parquet_events +POSTHOOK: type: QUERY +POSTHOOK: Input: default@parquet_events +#### A masked pattern was here #### +400 +PREHOOK: query: ALTER TABLE parquet_events ADD PARTITION (run_date=201209, game_id=39, event_name='hq_change') +#### A masked pattern was here #### +PREHOOK: type: ALTERTABLE_ADDPARTS +PREHOOK: Input: ### test.blobstore.path ###/parquet_nonstd_partitions_loc/parquet_nonstd_loc/ns-part-2 +PREHOOK: Output: default@parquet_events +POSTHOOK: query: ALTER TABLE parquet_events ADD PARTITION (run_date=201209, game_id=39, event_name='hq_change') +#### A masked pattern was here #### +POSTHOOK: type: ALTERTABLE_ADDPARTS +POSTHOOK: Input: ### test.blobstore.path ###/parquet_nonstd_partitions_loc/parquet_nonstd_loc/ns-part-2 +POSTHOOK: Output: default@parquet_events +POSTHOOK: Output: default@parquet_events@run_date=201209/game_id=39/event_name=hq_change +PREHOOK: query: INSERT OVERWRITE TABLE parquet_events PARTITION (run_date=201209, game_id=39, event_name='hq_change') +SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events +WHERE SUBSTR(run_date,1,6)='201209' +PREHOOK: type: QUERY +PREHOOK: Input: default@src_events +PREHOOK: Output: default@parquet_events@run_date=201209/game_id=39/event_name=hq_change +POSTHOOK: query: INSERT OVERWRITE TABLE parquet_events PARTITION (run_date=201209, game_id=39, event_name='hq_change') +SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events +WHERE SUBSTR(run_date,1,6)='201209' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src_events +POSTHOOK: Output: default@parquet_events@run_date=201209/game_id=39/event_name=hq_change +POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).event_data SIMPLE [(src_events)src_events.FieldSchema(name:event_data, type:string, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).full_uid SIMPLE [(src_events)src_events.FieldSchema(name:full_uid, type:bigint, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).log_id SIMPLE [(src_events)src_events.FieldSchema(name:log_id, type:bigint, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).session_id SIMPLE [(src_events)src_events.FieldSchema(name:session_id, type:string, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).time SIMPLE [(src_events)src_events.FieldSchema(name:time, type:bigint, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).type SIMPLE [(src_events)src_events.FieldSchema(name:type, type:int, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).uid SIMPLE [(src_events)src_events.FieldSchema(name:uid, type:bigint, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).user_id SIMPLE [(src_events)src_events.FieldSchema(name:user_id, type:bigint, comment:null), ] +PREHOOK: query: INSERT INTO TABLE parquet_events PARTITION (run_date=201209, game_id=39, event_name='hq_change') +SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events +WHERE SUBSTR(run_date,1,6)='201209' +PREHOOK: type: QUERY +PREHOOK: Input: default@src_events +PREHOOK: Output: default@parquet_events@run_date=201209/game_id=39/event_name=hq_change +POSTHOOK: query: INSERT INTO TABLE parquet_events PARTITION (run_date=201209, game_id=39, event_name='hq_change') +SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events +WHERE SUBSTR(run_date,1,6)='201209' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src_events +POSTHOOK: Output: default@parquet_events@run_date=201209/game_id=39/event_name=hq_change +POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).event_data SIMPLE [(src_events)src_events.FieldSchema(name:event_data, type:string, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).full_uid SIMPLE [(src_events)src_events.FieldSchema(name:full_uid, type:bigint, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).log_id SIMPLE [(src_events)src_events.FieldSchema(name:log_id, type:bigint, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).session_id SIMPLE [(src_events)src_events.FieldSchema(name:session_id, type:string, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).time SIMPLE [(src_events)src_events.FieldSchema(name:time, type:bigint, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).type SIMPLE [(src_events)src_events.FieldSchema(name:type, type:int, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).uid SIMPLE [(src_events)src_events.FieldSchema(name:uid, type:bigint, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=201209,game_id=39,event_name=hq_change).user_id SIMPLE [(src_events)src_events.FieldSchema(name:user_id, type:bigint, comment:null), ] +PREHOOK: query: SHOW PARTITIONS parquet_events +PREHOOK: type: SHOWPARTITIONS +PREHOOK: Input: default@parquet_events +POSTHOOK: query: SHOW PARTITIONS parquet_events +POSTHOOK: type: SHOWPARTITIONS +POSTHOOK: Input: default@parquet_events +run_date=201209/game_id=39/event_name=hq_change +run_date=20120921/game_id=39/event_name=hq_change +run_date=20121021/game_id=39/event_name=hq_change +run_date=201211/game_id=39/event_name=hq_change +run_date=20121121/game_id=39/event_name=hq_change +PREHOOK: query: SELECT COUNT(*) FROM parquet_events +PREHOOK: type: QUERY +PREHOOK: Input: default@parquet_events +#### A masked pattern was here #### +POSTHOOK: query: SELECT COUNT(*) FROM parquet_events +POSTHOOK: type: QUERY +POSTHOOK: Input: default@parquet_events +#### A masked pattern was here #### +500 +PREHOOK: query: INSERT OVERWRITE TABLE parquet_events PARTITION (run_date, game_id, event_name) +SELECT * FROM src_events +PREHOOK: type: QUERY +PREHOOK: Input: default@src_events +PREHOOK: Output: default@parquet_events +POSTHOOK: query: INSERT OVERWRITE TABLE parquet_events PARTITION (run_date, game_id, event_name) +SELECT * FROM src_events +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src_events +POSTHOOK: Output: default@parquet_events@run_date=20120921/game_id=39/event_name=hq_change +POSTHOOK: Output: default@parquet_events@run_date=20121021/game_id=39/event_name=hq_change +POSTHOOK: Output: default@parquet_events@run_date=20121121/game_id=39/event_name=hq_change +POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).event_data SIMPLE [(src_events)src_events.FieldSchema(name:event_data, type:string, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).full_uid SIMPLE [(src_events)src_events.FieldSchema(name:full_uid, type:bigint, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).log_id SIMPLE [(src_events)src_events.FieldSchema(name:log_id, type:bigint, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).session_id SIMPLE [(src_events)src_events.FieldSchema(name:session_id, type:string, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).time SIMPLE [(src_events)src_events.FieldSchema(name:time, type:bigint, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).type SIMPLE [(src_events)src_events.FieldSchema(name:type, type:int, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).uid SIMPLE [(src_events)src_events.FieldSchema(name:uid, type:bigint, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).user_id SIMPLE [(src_events)src_events.FieldSchema(name:user_id, type:bigint, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).event_data SIMPLE [(src_events)src_events.FieldSchema(name:event_data, type:string, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).full_uid SIMPLE [(src_events)src_events.FieldSchema(name:full_uid, type:bigint, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).log_id SIMPLE [(src_events)src_events.FieldSchema(name:log_id, type:bigint, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).session_id SIMPLE [(src_events)src_events.FieldSchema(name:session_id, type:string, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).time SIMPLE [(src_events)src_events.FieldSchema(name:time, type:bigint, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).type SIMPLE [(src_events)src_events.FieldSchema(name:type, type:int, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).uid SIMPLE [(src_events)src_events.FieldSchema(name:uid, type:bigint, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).user_id SIMPLE [(src_events)src_events.FieldSchema(name:user_id, type:bigint, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).event_data SIMPLE [(src_events)src_events.FieldSchema(name:event_data, type:string, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).full_uid SIMPLE [(src_events)src_events.FieldSchema(name:full_uid, type:bigint, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).log_id SIMPLE [(src_events)src_events.FieldSchema(name:log_id, type:bigint, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).session_id SIMPLE [(src_events)src_events.FieldSchema(name:session_id, type:string, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).time SIMPLE [(src_events)src_events.FieldSchema(name:time, type:bigint, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).type SIMPLE [(src_events)src_events.FieldSchema(name:type, type:int, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).uid SIMPLE [(src_events)src_events.FieldSchema(name:uid, type:bigint, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).user_id SIMPLE [(src_events)src_events.FieldSchema(name:user_id, type:bigint, comment:null), ] +PREHOOK: query: SHOW PARTITIONS parquet_events +PREHOOK: type: SHOWPARTITIONS +PREHOOK: Input: default@parquet_events +POSTHOOK: query: SHOW PARTITIONS parquet_events +POSTHOOK: type: SHOWPARTITIONS +POSTHOOK: Input: default@parquet_events +run_date=201209/game_id=39/event_name=hq_change +run_date=20120921/game_id=39/event_name=hq_change +run_date=20121021/game_id=39/event_name=hq_change +run_date=201211/game_id=39/event_name=hq_change +run_date=20121121/game_id=39/event_name=hq_change +PREHOOK: query: SELECT COUNT(*) FROM parquet_events +PREHOOK: type: QUERY +PREHOOK: Input: default@parquet_events +#### A masked pattern was here #### +POSTHOOK: query: SELECT COUNT(*) FROM parquet_events +POSTHOOK: type: QUERY +POSTHOOK: Input: default@parquet_events +#### A masked pattern was here #### +500 +PREHOOK: query: ALTER TABLE parquet_events ADD PARTITION (run_date=201207, game_id=39, event_name='hq_change') +#### A masked pattern was here #### +PREHOOK: type: ALTERTABLE_ADDPARTS +PREHOOK: Input: ### test.blobstore.path ###/parquet_nonstd_partitions_loc/parquet_nonstd_loc/ns-part-3 +PREHOOK: Output: default@parquet_events +POSTHOOK: query: ALTER TABLE parquet_events ADD PARTITION (run_date=201207, game_id=39, event_name='hq_change') +#### A masked pattern was here #### +POSTHOOK: type: ALTERTABLE_ADDPARTS +POSTHOOK: Input: ### test.blobstore.path ###/parquet_nonstd_partitions_loc/parquet_nonstd_loc/ns-part-3 +POSTHOOK: Output: default@parquet_events +POSTHOOK: Output: default@parquet_events@run_date=201207/game_id=39/event_name=hq_change +PREHOOK: query: INSERT INTO TABLE parquet_events PARTITION (run_date=201207, game_id=39, event_name='hq_change') +SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events +WHERE SUBSTR(run_date,1,6)='201209' +PREHOOK: type: QUERY +PREHOOK: Input: default@src_events +PREHOOK: Output: default@parquet_events@run_date=201207/game_id=39/event_name=hq_change +POSTHOOK: query: INSERT INTO TABLE parquet_events PARTITION (run_date=201207, game_id=39, event_name='hq_change') +SELECT log_id,`time`,uid,user_id,type,event_data,session_id,full_uid FROM src_events +WHERE SUBSTR(run_date,1,6)='201209' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src_events +POSTHOOK: Output: default@parquet_events@run_date=201207/game_id=39/event_name=hq_change +POSTHOOK: Lineage: parquet_events PARTITION(run_date=201207,game_id=39,event_name=hq_change).event_data SIMPLE [(src_events)src_events.FieldSchema(name:event_data, type:string, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=201207,game_id=39,event_name=hq_change).full_uid SIMPLE [(src_events)src_events.FieldSchema(name:full_uid, type:bigint, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=201207,game_id=39,event_name=hq_change).log_id SIMPLE [(src_events)src_events.FieldSchema(name:log_id, type:bigint, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=201207,game_id=39,event_name=hq_change).session_id SIMPLE [(src_events)src_events.FieldSchema(name:session_id, type:string, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=201207,game_id=39,event_name=hq_change).time SIMPLE [(src_events)src_events.FieldSchema(name:time, type:bigint, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=201207,game_id=39,event_name=hq_change).type SIMPLE [(src_events)src_events.FieldSchema(name:type, type:int, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=201207,game_id=39,event_name=hq_change).uid SIMPLE [(src_events)src_events.FieldSchema(name:uid, type:bigint, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=201207,game_id=39,event_name=hq_change).user_id SIMPLE [(src_events)src_events.FieldSchema(name:user_id, type:bigint, comment:null), ] +PREHOOK: query: SHOW PARTITIONS parquet_events +PREHOOK: type: SHOWPARTITIONS +PREHOOK: Input: default@parquet_events +POSTHOOK: query: SHOW PARTITIONS parquet_events +POSTHOOK: type: SHOWPARTITIONS +POSTHOOK: Input: default@parquet_events +run_date=201207/game_id=39/event_name=hq_change +run_date=201209/game_id=39/event_name=hq_change +run_date=20120921/game_id=39/event_name=hq_change +run_date=20121021/game_id=39/event_name=hq_change +run_date=201211/game_id=39/event_name=hq_change +run_date=20121121/game_id=39/event_name=hq_change +PREHOOK: query: SELECT COUNT(*) FROM parquet_events +PREHOOK: type: QUERY +PREHOOK: Input: default@parquet_events +PREHOOK: Input: default@parquet_events@run_date=201207/game_id=39/event_name=hq_change +PREHOOK: Input: default@parquet_events@run_date=201209/game_id=39/event_name=hq_change +PREHOOK: Input: default@parquet_events@run_date=20120921/game_id=39/event_name=hq_change +PREHOOK: Input: default@parquet_events@run_date=20121021/game_id=39/event_name=hq_change +PREHOOK: Input: default@parquet_events@run_date=201211/game_id=39/event_name=hq_change +PREHOOK: Input: default@parquet_events@run_date=20121121/game_id=39/event_name=hq_change +#### A masked pattern was here #### +POSTHOOK: query: SELECT COUNT(*) FROM parquet_events +POSTHOOK: type: QUERY +POSTHOOK: Input: default@parquet_events +POSTHOOK: Input: default@parquet_events@run_date=201207/game_id=39/event_name=hq_change +POSTHOOK: Input: default@parquet_events@run_date=201209/game_id=39/event_name=hq_change +POSTHOOK: Input: default@parquet_events@run_date=20120921/game_id=39/event_name=hq_change +POSTHOOK: Input: default@parquet_events@run_date=20121021/game_id=39/event_name=hq_change +POSTHOOK: Input: default@parquet_events@run_date=201211/game_id=39/event_name=hq_change +POSTHOOK: Input: default@parquet_events@run_date=20121121/game_id=39/event_name=hq_change +#### A masked pattern was here #### +550 +PREHOOK: query: INSERT OVERWRITE TABLE parquet_events PARTITION (run_date, game_id, event_name) +SELECT * FROM src_events +PREHOOK: type: QUERY +PREHOOK: Input: default@src_events +PREHOOK: Output: default@parquet_events +POSTHOOK: query: INSERT OVERWRITE TABLE parquet_events PARTITION (run_date, game_id, event_name) +SELECT * FROM src_events +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src_events +POSTHOOK: Output: default@parquet_events@run_date=20120921/game_id=39/event_name=hq_change +POSTHOOK: Output: default@parquet_events@run_date=20121021/game_id=39/event_name=hq_change +POSTHOOK: Output: default@parquet_events@run_date=20121121/game_id=39/event_name=hq_change +POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).event_data SIMPLE [(src_events)src_events.FieldSchema(name:event_data, type:string, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).full_uid SIMPLE [(src_events)src_events.FieldSchema(name:full_uid, type:bigint, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).log_id SIMPLE [(src_events)src_events.FieldSchema(name:log_id, type:bigint, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).session_id SIMPLE [(src_events)src_events.FieldSchema(name:session_id, type:string, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).time SIMPLE [(src_events)src_events.FieldSchema(name:time, type:bigint, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).type SIMPLE [(src_events)src_events.FieldSchema(name:type, type:int, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).uid SIMPLE [(src_events)src_events.FieldSchema(name:uid, type:bigint, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=20120921,game_id=39,event_name=hq_change).user_id SIMPLE [(src_events)src_events.FieldSchema(name:user_id, type:bigint, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).event_data SIMPLE [(src_events)src_events.FieldSchema(name:event_data, type:string, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).full_uid SIMPLE [(src_events)src_events.FieldSchema(name:full_uid, type:bigint, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).log_id SIMPLE [(src_events)src_events.FieldSchema(name:log_id, type:bigint, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).session_id SIMPLE [(src_events)src_events.FieldSchema(name:session_id, type:string, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).time SIMPLE [(src_events)src_events.FieldSchema(name:time, type:bigint, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).type SIMPLE [(src_events)src_events.FieldSchema(name:type, type:int, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).uid SIMPLE [(src_events)src_events.FieldSchema(name:uid, type:bigint, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121021,game_id=39,event_name=hq_change).user_id SIMPLE [(src_events)src_events.FieldSchema(name:user_id, type:bigint, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).event_data SIMPLE [(src_events)src_events.FieldSchema(name:event_data, type:string, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).full_uid SIMPLE [(src_events)src_events.FieldSchema(name:full_uid, type:bigint, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).log_id SIMPLE [(src_events)src_events.FieldSchema(name:log_id, type:bigint, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).session_id SIMPLE [(src_events)src_events.FieldSchema(name:session_id, type:string, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).time SIMPLE [(src_events)src_events.FieldSchema(name:time, type:bigint, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).type SIMPLE [(src_events)src_events.FieldSchema(name:type, type:int, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).uid SIMPLE [(src_events)src_events.FieldSchema(name:uid, type:bigint, comment:null), ] +POSTHOOK: Lineage: parquet_events PARTITION(run_date=20121121,game_id=39,event_name=hq_change).user_id SIMPLE [(src_events)src_events.FieldSchema(name:user_id, type:bigint, comment:null), ] +PREHOOK: query: SHOW PARTITIONS parquet_events +PREHOOK: type: SHOWPARTITIONS +PREHOOK: Input: default@parquet_events +POSTHOOK: query: SHOW PARTITIONS parquet_events +POSTHOOK: type: SHOWPARTITIONS +POSTHOOK: Input: default@parquet_events +run_date=201207/game_id=39/event_name=hq_change +run_date=201209/game_id=39/event_name=hq_change +run_date=20120921/game_id=39/event_name=hq_change +run_date=20121021/game_id=39/event_name=hq_change +run_date=201211/game_id=39/event_name=hq_change +run_date=20121121/game_id=39/event_name=hq_change +PREHOOK: query: SELECT COUNT(*) FROM parquet_events +PREHOOK: type: QUERY +PREHOOK: Input: default@parquet_events +PREHOOK: Input: default@parquet_events@run_date=201207/game_id=39/event_name=hq_change +PREHOOK: Input: default@parquet_events@run_date=201209/game_id=39/event_name=hq_change +PREHOOK: Input: default@parquet_events@run_date=20120921/game_id=39/event_name=hq_change +PREHOOK: Input: default@parquet_events@run_date=20121021/game_id=39/event_name=hq_change +PREHOOK: Input: default@parquet_events@run_date=201211/game_id=39/event_name=hq_change +PREHOOK: Input: default@parquet_events@run_date=20121121/game_id=39/event_name=hq_change +#### A masked pattern was here #### +POSTHOOK: query: SELECT COUNT(*) FROM parquet_events +POSTHOOK: type: QUERY +POSTHOOK: Input: default@parquet_events +POSTHOOK: Input: default@parquet_events@run_date=201207/game_id=39/event_name=hq_change +POSTHOOK: Input: default@parquet_events@run_date=201209/game_id=39/event_name=hq_change +POSTHOOK: Input: default@parquet_events@run_date=20120921/game_id=39/event_name=hq_change +POSTHOOK: Input: default@parquet_events@run_date=20121021/game_id=39/event_name=hq_change +POSTHOOK: Input: default@parquet_events@run_date=201211/game_id=39/event_name=hq_change +POSTHOOK: Input: default@parquet_events@run_date=20121121/game_id=39/event_name=hq_change +#### A masked pattern was here #### +550 +PREHOOK: query: ALTER TABLE parquet_events DROP PARTITION (run_date=201211, game_id=39, event_name='hq_change') +PREHOOK: type: ALTERTABLE_DROPPARTS +PREHOOK: Input: default@parquet_events +PREHOOK: Output: default@parquet_events@run_date=201211/game_id=39/event_name=hq_change +POSTHOOK: query: ALTER TABLE parquet_events DROP PARTITION (run_date=201211, game_id=39, event_name='hq_change') +POSTHOOK: type: ALTERTABLE_DROPPARTS +POSTHOOK: Input: default@parquet_events +POSTHOOK: Output: default@parquet_events@run_date=201211/game_id=39/event_name=hq_change +PREHOOK: query: ALTER TABLE parquet_events DROP PARTITION (run_date=201209, game_id=39, event_name='hq_change') +PREHOOK: type: ALTERTABLE_DROPPARTS +PREHOOK: Input: default@parquet_events +PREHOOK: Output: default@parquet_events@run_date=201209/game_id=39/event_name=hq_change +POSTHOOK: query: ALTER TABLE parquet_events DROP PARTITION (run_date=201209, game_id=39, event_name='hq_change') +POSTHOOK: type: ALTERTABLE_DROPPARTS +POSTHOOK: Input: default@parquet_events +POSTHOOK: Output: default@parquet_events@run_date=201209/game_id=39/event_name=hq_change +PREHOOK: query: ALTER TABLE parquet_events DROP PARTITION (run_date=201207, game_id=39, event_name='hq_change') +PREHOOK: type: ALTERTABLE_DROPPARTS +PREHOOK: Input: default@parquet_events +PREHOOK: Output: default@parquet_events@run_date=201207/game_id=39/event_name=hq_change +POSTHOOK: query: ALTER TABLE parquet_events DROP PARTITION (run_date=201207, game_id=39, event_name='hq_change') +POSTHOOK: type: ALTERTABLE_DROPPARTS +POSTHOOK: Input: default@parquet_events +POSTHOOK: Output: default@parquet_events@run_date=201207/game_id=39/event_name=hq_change +PREHOOK: query: SHOW PARTITIONS parquet_events +PREHOOK: type: SHOWPARTITIONS +PREHOOK: Input: default@parquet_events +POSTHOOK: query: SHOW PARTITIONS parquet_events +POSTHOOK: type: SHOWPARTITIONS +POSTHOOK: Input: default@parquet_events +run_date=20120921/game_id=39/event_name=hq_change +run_date=20121021/game_id=39/event_name=hq_change +run_date=20121121/game_id=39/event_name=hq_change +PREHOOK: query: SELECT COUNT(*) FROM parquet_events +PREHOOK: type: QUERY +PREHOOK: Input: default@parquet_events +#### A masked pattern was here #### +POSTHOOK: query: SELECT COUNT(*) FROM parquet_events +POSTHOOK: type: QUERY +POSTHOOK: Input: default@parquet_events +#### A masked pattern was here #### +200 diff --git a/itests/hive-blobstore/src/test/results/clientpositive/ptf_parquetfile.q.out b/itests/hive-blobstore/src/test/results/clientpositive/ptf_parquetfile.q.out new file mode 100644 index 0000000000..192b80f0ac --- /dev/null +++ b/itests/hive-blobstore/src/test/results/clientpositive/ptf_parquetfile.q.out @@ -0,0 +1,104 @@ +PREHOOK: query: DROP TABLE part_parquet +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE part_parquet +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE part_parquet( + p_partkey int, + p_name string, + p_mfgr string, + p_brand string, + p_type string, + p_size int, + p_container string, + p_retailprice double, + p_comment string +) +STORED AS PARQUET +#### A masked pattern was here #### +PREHOOK: type: CREATETABLE +PREHOOK: Input: ### test.blobstore.path ###/ptf_parquetfile/part_parquet +PREHOOK: Output: database:default +PREHOOK: Output: default@part_parquet +POSTHOOK: query: CREATE TABLE part_parquet( + p_partkey int, + p_name string, + p_mfgr string, + p_brand string, + p_type string, + p_size int, + p_container string, + p_retailprice double, + p_comment string +) +STORED AS PARQUET +#### A masked pattern was here #### +POSTHOOK: type: CREATETABLE +POSTHOOK: Input: ### test.blobstore.path ###/ptf_parquetfile/part_parquet +POSTHOOK: Output: database:default +POSTHOOK: Output: default@part_parquet +PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/part.parquet' OVERWRITE INTO TABLE part_parquet +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@part_parquet +POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/part.parquet' OVERWRITE INTO TABLE part_parquet +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@part_parquet +PREHOOK: query: SELECT + p_mfgr, p_name, p_size, + RANK() OVER (PARTITION BY p_mfgr ORDER BY p_name) AS r, + DENSE_RANK() OVER (PARTITION BY p_mfgr ORDER BY p_name) AS dr, + SUM(p_retailprice) OVER ( + PARTITION BY p_mfgr + ORDER BY p_name ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW + ) AS s1 +FROM NOOP( + ON part_parquet + PARTITION BY p_mfgr + ORDER BY p_name +) +PREHOOK: type: QUERY +PREHOOK: Input: default@part_parquet +#### A masked pattern was here #### +POSTHOOK: query: SELECT + p_mfgr, p_name, p_size, + RANK() OVER (PARTITION BY p_mfgr ORDER BY p_name) AS r, + DENSE_RANK() OVER (PARTITION BY p_mfgr ORDER BY p_name) AS dr, + SUM(p_retailprice) OVER ( + PARTITION BY p_mfgr + ORDER BY p_name ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW + ) AS s1 +FROM NOOP( + ON part_parquet + PARTITION BY p_mfgr + ORDER BY p_name +) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part_parquet +#### A masked pattern was here #### +Manufacturer#1 almond antique burnished rose metallic 2 1 1 1173.15 +Manufacturer#1 almond antique burnished rose metallic 2 1 1 2346.3 +Manufacturer#1 almond antique chartreuse lavender yellow 34 3 2 4100.06 +Manufacturer#1 almond antique salmon chartreuse burlywood 6 4 3 5702.650000000001 +Manufacturer#1 almond aquamarine burnished black steel 28 5 4 7117.070000000001 +Manufacturer#1 almond aquamarine pink moccasin thistle 42 6 5 8749.730000000001 +Manufacturer#2 almond antique violet chocolate turquoise 14 1 1 1690.68 +Manufacturer#2 almond antique violet turquoise frosted 40 2 2 3491.38 +Manufacturer#2 almond aquamarine midnight light salmon 2 3 3 5523.360000000001 +Manufacturer#2 almond aquamarine rose maroon antique 25 4 4 7222.02 +Manufacturer#2 almond aquamarine sandy cyan gainsboro 18 5 5 8923.62 +Manufacturer#3 almond antique chartreuse khaki white 17 1 1 1671.68 +Manufacturer#3 almond antique forest lavender goldenrod 14 2 2 2861.95 +Manufacturer#3 almond antique metallic orange dim 19 3 3 4272.34 +Manufacturer#3 almond antique misty red olive 1 4 4 6195.32 +Manufacturer#3 almond antique olive coral navajo 45 5 5 7532.61 +Manufacturer#4 almond antique gainsboro frosted violet 10 1 1 1620.67 +Manufacturer#4 almond antique violet mint lemon 39 2 2 2996.09 +Manufacturer#4 almond aquamarine floral ivory bisque 27 3 3 4202.35 +Manufacturer#4 almond aquamarine yellow dodger mint 7 4 4 6047.27 +Manufacturer#4 almond azure aquamarine papaya violet 12 5 5 7337.620000000001 +Manufacturer#5 almond antique blue firebrick mint 31 1 1 1789.69 +Manufacturer#5 almond antique medium spring khaki 6 2 2 3401.3500000000004 +Manufacturer#5 almond antique sky peru orange 2 3 3 5190.08 +Manufacturer#5 almond aquamarine dodger light gainsboro 46 4 4 6208.18 +Manufacturer#5 almond azure blanched chiffon midnight 23 5 5 7672.66