From 1702299c0a43e458cd77b989a44f8b103c8a1e8b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20Mes=C3=9Fner?= Date: Fri, 6 Feb 2015 07:33:31 +0000 Subject: [PATCH] Add test case for parquet schema evolution --- .../clientpositive/parquet_schema_evolution.q | 30 ++++++ .../clientpositive/parquet_schema_evolution.q.out | 112 ++++++++++++++++++++ 2 files changed, 142 insertions(+) create mode 100644 ql/src/test/queries/clientpositive/parquet_schema_evolution.q create mode 100644 ql/src/test/results/clientpositive/parquet_schema_evolution.q.out diff --git a/ql/src/test/queries/clientpositive/parquet_schema_evolution.q b/ql/src/test/queries/clientpositive/parquet_schema_evolution.q new file mode 100644 index 0000000..e1446c3 --- /dev/null +++ b/ql/src/test/queries/clientpositive/parquet_schema_evolution.q @@ -0,0 +1,30 @@ +set hive.exec.dynamic.partition.mode=nonstrict; +set hive.exec.dynamic.partition=true; + +DROP TABLE parquet_schema_evolution_staging; +DROP TABLE parquet_schema_evolution; + +CREATE TABLE parquet_schema_evolution_staging ( + id int, + str string, + part string +) ROW FORMAT DELIMITED +FIELDS TERMINATED BY '|'; + +CREATE TABLE parquet_schema_evolution ( + id int, + str string +) PARTITIONED BY (part string) +STORED AS PARQUET; + +LOAD DATA LOCAL INPATH '../../data/files/parquet_partitioned.txt' OVERWRITE INTO TABLE parquet_schema_evolution_staging; + +INSERT OVERWRITE TABLE parquet_schema_evolution PARTITION (part) SELECT * FROM parquet_schema_evolution_staging; + +ALTER TABLE parquet_schema_evolution ADD COLUMNS (new string); + +-- returns nothing, but must not fail. +SELECT new FROM parquet_schema_evolution WHERE part='part1'; + +DROP TABLE parquet_schema_evolution_staging; +DROP TABLE parquet_schema_evolution; diff --git a/ql/src/test/results/clientpositive/parquet_schema_evolution.q.out b/ql/src/test/results/clientpositive/parquet_schema_evolution.q.out new file mode 100644 index 0000000..71a16c0 --- /dev/null +++ b/ql/src/test/results/clientpositive/parquet_schema_evolution.q.out @@ -0,0 +1,112 @@ +PREHOOK: query: DROP TABLE parquet_schema_evolution_staging +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE parquet_schema_evolution_staging +POSTHOOK: type: DROPTABLE +PREHOOK: query: DROP TABLE parquet_schema_evolution +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE parquet_schema_evolution +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE parquet_schema_evolution_staging ( + id int, + str string, + part string +) ROW FORMAT DELIMITED +FIELDS TERMINATED BY '|' +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +POSTHOOK: query: CREATE TABLE parquet_schema_evolution_staging ( + id int, + str string, + part string +) ROW FORMAT DELIMITED +FIELDS TERMINATED BY '|' +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@parquet_schema_evolution_staging +PREHOOK: query: CREATE TABLE parquet_schema_evolution ( + id int, + str string +) PARTITIONED BY (part string) +STORED AS PARQUET +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +POSTHOOK: query: CREATE TABLE parquet_schema_evolution ( + id int, + str string +) PARTITIONED BY (part string) +STORED AS PARQUET +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@parquet_schema_evolution +PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/parquet_partitioned.txt' OVERWRITE INTO TABLE parquet_schema_evolution_staging +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@parquet_schema_evolution_staging +POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/parquet_partitioned.txt' OVERWRITE INTO TABLE parquet_schema_evolution_staging +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@parquet_schema_evolution_staging +PREHOOK: query: INSERT OVERWRITE TABLE parquet_schema_evolution PARTITION (part) SELECT * FROM parquet_schema_evolution_staging +PREHOOK: type: QUERY +PREHOOK: Input: default@parquet_schema_evolution_staging +PREHOOK: Output: default@parquet_schema_evolution +POSTHOOK: query: INSERT OVERWRITE TABLE parquet_schema_evolution PARTITION (part) SELECT * FROM parquet_schema_evolution_staging +POSTHOOK: type: QUERY +POSTHOOK: Input: default@parquet_schema_evolution_staging +POSTHOOK: Output: default@parquet_schema_evolution@part=part1 +POSTHOOK: Output: default@parquet_schema_evolution@part=part2 +POSTHOOK: Lineage: parquet_schema_evolution PARTITION(part=part1).id SIMPLE [(parquet_schema_evolution_staging)parquet_schema_evolution_staging.FieldSchema(name:id, type:int, comment:null), ] +POSTHOOK: Lineage: parquet_schema_evolution PARTITION(part=part1).str SIMPLE [(parquet_schema_evolution_staging)parquet_schema_evolution_staging.FieldSchema(name:str, type:string, comment:null), ] +POSTHOOK: Lineage: parquet_schema_evolution PARTITION(part=part2).id SIMPLE [(parquet_schema_evolution_staging)parquet_schema_evolution_staging.FieldSchema(name:id, type:int, comment:null), ] +POSTHOOK: Lineage: parquet_schema_evolution PARTITION(part=part2).str SIMPLE [(parquet_schema_evolution_staging)parquet_schema_evolution_staging.FieldSchema(name:str, type:string, comment:null), ] +PREHOOK: query: ALTER TABLE parquet_schema_evolution ADD COLUMNS (new string) +PREHOOK: type: ALTERTABLE_ADDCOLS +PREHOOK: Input: default@parquet_schema_evolution +PREHOOK: Output: default@parquet_schema_evolution +POSTHOOK: query: ALTER TABLE parquet_schema_evolution ADD COLUMNS (new string) +POSTHOOK: type: ALTERTABLE_ADDCOLS +POSTHOOK: Input: default@parquet_schema_evolution +POSTHOOK: Output: default@parquet_schema_evolution +POSTHOOK: Lineage: parquet_schema_evolution PARTITION(part=part1).id SIMPLE [(parquet_schema_evolution_staging)parquet_schema_evolution_staging.FieldSchema(name:id, type:int, comment:null), ] +POSTHOOK: Lineage: parquet_schema_evolution PARTITION(part=part1).str SIMPLE [(parquet_schema_evolution_staging)parquet_schema_evolution_staging.FieldSchema(name:str, type:string, comment:null), ] +POSTHOOK: Lineage: parquet_schema_evolution PARTITION(part=part2).id SIMPLE [(parquet_schema_evolution_staging)parquet_schema_evolution_staging.FieldSchema(name:id, type:int, comment:null), ] +POSTHOOK: Lineage: parquet_schema_evolution PARTITION(part=part2).str SIMPLE [(parquet_schema_evolution_staging)parquet_schema_evolution_staging.FieldSchema(name:str, type:string, comment:null), ] +PREHOOK: query: SELECT new FROM parquet_schema_evolution WHERE part='part1' +PREHOOK: type: QUERY +PREHOOK: Input: default@parquet_schema_evolution +PREHOOK: Input: default@parquet_schema_evolution@part=part1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT new FROM parquet_schema_evolution WHERE part='part1' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@parquet_schema_evolution +POSTHOOK: Input: default@parquet_schema_evolution@part=part1 +#### A masked pattern was here #### +POSTHOOK: Lineage: parquet_schema_evolution PARTITION(part=part1).id SIMPLE [(parquet_schema_evolution_staging)parquet_schema_evolution_staging.FieldSchema(name:id, type:int, comment:null), ] +POSTHOOK: Lineage: parquet_schema_evolution PARTITION(part=part1).str SIMPLE [(parquet_schema_evolution_staging)parquet_schema_evolution_staging.FieldSchema(name:str, type:string, comment:null), ] +POSTHOOK: Lineage: parquet_schema_evolution PARTITION(part=part2).id SIMPLE [(parquet_schema_evolution_staging)parquet_schema_evolution_staging.FieldSchema(name:id, type:int, comment:null), ] +POSTHOOK: Lineage: parquet_schema_evolution PARTITION(part=part2).str SIMPLE [(parquet_schema_evolution_staging)parquet_schema_evolution_staging.FieldSchema(name:str, type:string, comment:null), ] +NULL +PREHOOK: query: DROP TABLE parquet_schema_evolution_staging +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@parquet_schema_evolution_staging +PREHOOK: Output: default@parquet_schema_evolution_staging +POSTHOOK: query: DROP TABLE parquet_schema_evolution_staging +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@parquet_schema_evolution_staging +POSTHOOK: Output: default@parquet_schema_evolution_staging +POSTHOOK: Lineage: parquet_schema_evolution PARTITION(part=part1).id SIMPLE [(parquet_schema_evolution_staging)parquet_schema_evolution_staging.FieldSchema(name:id, type:int, comment:null), ] +POSTHOOK: Lineage: parquet_schema_evolution PARTITION(part=part1).str SIMPLE [(parquet_schema_evolution_staging)parquet_schema_evolution_staging.FieldSchema(name:str, type:string, comment:null), ] +POSTHOOK: Lineage: parquet_schema_evolution PARTITION(part=part2).id SIMPLE [(parquet_schema_evolution_staging)parquet_schema_evolution_staging.FieldSchema(name:id, type:int, comment:null), ] +POSTHOOK: Lineage: parquet_schema_evolution PARTITION(part=part2).str SIMPLE [(parquet_schema_evolution_staging)parquet_schema_evolution_staging.FieldSchema(name:str, type:string, comment:null), ] +PREHOOK: query: DROP TABLE parquet_schema_evolution +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@parquet_schema_evolution +PREHOOK: Output: default@parquet_schema_evolution +POSTHOOK: query: DROP TABLE parquet_schema_evolution +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@parquet_schema_evolution +POSTHOOK: Output: default@parquet_schema_evolution +POSTHOOK: Lineage: parquet_schema_evolution PARTITION(part=part1).id SIMPLE [(parquet_schema_evolution_staging)parquet_schema_evolution_staging.FieldSchema(name:id, type:int, comment:null), ] +POSTHOOK: Lineage: parquet_schema_evolution PARTITION(part=part1).str SIMPLE [(parquet_schema_evolution_staging)parquet_schema_evolution_staging.FieldSchema(name:str, type:string, comment:null), ] +POSTHOOK: Lineage: parquet_schema_evolution PARTITION(part=part2).id SIMPLE [(parquet_schema_evolution_staging)parquet_schema_evolution_staging.FieldSchema(name:id, type:int, comment:null), ] +POSTHOOK: Lineage: parquet_schema_evolution PARTITION(part=part2).str SIMPLE [(parquet_schema_evolution_staging)parquet_schema_evolution_staging.FieldSchema(name:str, type:string, comment:null), ] -- 1.7.9.5