diff --git a/ql/src/test/queries/clientpositive/avro_comments.q b/ql/src/test/queries/clientpositive/avro_comments.q new file mode 100644 index 0000000000000000000000000000000000000000..f55196497d279447868c3c8350aa1fcb4ee223f1 --- /dev/null +++ b/ql/src/test/queries/clientpositive/avro_comments.q @@ -0,0 +1,150 @@ +-- verify Avro columns comments +DROP TABLE IF EXISTS testAvroComments1; + +CREATE TABLE testAvroComments1 +ROW FORMAT +SERDE 'org.apache.hadoop.hive.serde2.avro.AvroSerDe' +STORED AS +INPUTFORMAT 'org.apache.hadoop.hive.ql.io.avro.AvroContainerInputFormat' +OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.avro.AvroContainerOutputFormat' +TBLPROPERTIES ('avro.schema.literal'='{ + "namespace": "testing.hive.avro.serde", + "name": "doctors", + "type": "record", + "fields": [ + { + "name":"number", + "type":"int", + "doc":"Order of playing the role" + }, + { + "name":"first_name", + "type":"string", + "doc":"first name of actor playing role" + }, + { + "name":"last_name", + "type":"string", + "doc":"last name of actor playing role" + }, + { + "name":"extra_field", + "type":"string", + "doc":"an extra field not in the original file", + "default":"fishfingers and custard" + } + ] +}'); + +DESCRIBE testAvroComments1; +DROP TABLE testAvroComments1; + +DROP TABLE IF EXISTS testAvroComments2; +CREATE TABLE testAvroComments2 +ROW FORMAT +SERDE 'org.apache.hadoop.hive.serde2.avro.AvroSerDe' +STORED AS +INPUTFORMAT 'org.apache.hadoop.hive.ql.io.avro.AvroContainerInputFormat' +OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.avro.AvroContainerOutputFormat' +TBLPROPERTIES ('avro.schema.literal'='{ + "namespace": "testing.hive.avro.serde", + "name": "doctors", + "type": "record", + "fields": [ + { + "name":"number", + "type":"int", + "doc":"Order of playing the role" + }, + { + "name":"first_name", + "type":"string" + }, + { + "name":"last_name", + "type":"string", + "doc":"last name of actor playing role" + }, + { + "name":"extra_field", + "type":"string", + "default":"fishfingers and custard" + } + ] +}'); + +DESCRIBE testAvroComments2; +DROP TABLE testAvroComments2; + +DROP TABLE IF EXISTS testAvroComments3; +CREATE TABLE testAvroComments3 +ROW FORMAT +SERDE 'org.apache.hadoop.hive.serde2.avro.AvroSerDe' +STORED AS +INPUTFORMAT 'org.apache.hadoop.hive.ql.io.avro.AvroContainerInputFormat' +OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.avro.AvroContainerOutputFormat' +TBLPROPERTIES ('avro.schema.literal'='{ + "namespace": "testing.hive.avro.serde", + "name": "doctors", + "type": "record", + "fields": [ + { + "name":"number", + "type":"int" + }, + { + "name":"first_name", + "type":"string" + }, + { + "name":"last_name", + "type":"string" + }, + { + "name":"extra_field", + "type":"string", + "default":"fishfingers and custard" + } + ] +}'); + +DESCRIBE testAvroComments3; +DROP TABLE testAvroComments3; + +DROP TABLE IF EXISTS testAvroComments4; + +CREATE TABLE testAvroComments4 ( + number int COMMENT "Order of playing the role", + first_name string COMMENT "first name of actor playing role", + last_name string COMMENT "last name of actor playing role", + extra_field string COMMENT "an extra field not in the original file") +STORED AS AVRO; + +DESCRIBE testAvroComments4; +DROP TABLE testAvroComments4; + +DROP TABLE IF EXISTS testAvroComments5; + +CREATE TABLE testAvroComments5 ( + number int COMMENT "Order of playing the role", + first_name string, + last_name string COMMENT "last name of actor playing role", + extra_field string) +STORED AS AVRO; + +DESCRIBE testAvroComments5; +DROP TABLE testAvroComments5; + +DROP TABLE IF EXISTS testAvroComments6; + +CREATE TABLE testAvroComments6 ( + number int, + first_name string, + last_name string, + extra_field string) +STORED AS AVRO; + +DESCRIBE testAvroComments6; +DROP TABLE testAvroComments6; + + diff --git a/ql/src/test/queries/clientpositive/avro_compression_enabled.q b/ql/src/test/queries/clientpositive/avro_compression_enabled.q index cb6f173ccfa76420f07c7d7e69749482537127aa..58867ff50fcfd4a78133ea6c8284241b2883d464 100644 --- a/ql/src/test/queries/clientpositive/avro_compression_enabled.q +++ b/ql/src/test/queries/clientpositive/avro_compression_enabled.q @@ -29,7 +29,7 @@ TBLPROPERTIES ('avro.schema.literal'='{ { "name":"extra_field", "type":"string", - "doc:":"an extra field not in the original file", + "doc":"an extra field not in the original file", "default":"fishfingers and custard" } ] diff --git a/ql/src/test/queries/clientpositive/avro_evolved_schemas.q b/ql/src/test/queries/clientpositive/avro_evolved_schemas.q index 59cb21053fbb366962e39126f4f6623b8d53c8e0..05be324453ec0b6d86b7d77d7c7670ce390cbcd4 100644 --- a/ql/src/test/queries/clientpositive/avro_evolved_schemas.q +++ b/ql/src/test/queries/clientpositive/avro_evolved_schemas.q @@ -30,7 +30,7 @@ TBLPROPERTIES ('avro.schema.literal'='{ { "name":"extra_field", "type":"string", - "doc:":"an extra field not in the original file", + "doc":"an extra field not in the original file", "default":"fishfingers and custard" } ] diff --git a/ql/src/test/queries/clientpositive/avro_joins.q b/ql/src/test/queries/clientpositive/avro_joins.q index 6d79bf1c1b4b403b6f33047dbe56ad656a68f990..9d06881e4965d54d5b15d2f7ce3adba1bc51e81c 100644 --- a/ql/src/test/queries/clientpositive/avro_joins.q +++ b/ql/src/test/queries/clientpositive/avro_joins.q @@ -31,7 +31,7 @@ TBLPROPERTIES ('avro.schema.literal'='{ { "name":"extra_field", "type":"string", - "doc:":"an extra field not in the original file", + "doc":"an extra field not in the original file", "default":"fishfingers and custard" } ] diff --git a/ql/src/test/results/clientpositive/avro_add_column.q.out b/ql/src/test/results/clientpositive/avro_add_column.q.out index 3aae56225ad671e44291d566b4a2400a9ee4ead9..5e448424c9ec02950a32d5e6166ab34e040c57ef 100644 --- a/ql/src/test/results/clientpositive/avro_add_column.q.out +++ b/ql/src/test/results/clientpositive/avro_add_column.q.out @@ -24,8 +24,8 @@ PREHOOK: Input: default@doctors POSTHOOK: query: DESCRIBE doctors POSTHOOK: type: DESCTABLE POSTHOOK: Input: default@doctors -number int from deserializer -first_name string from deserializer +number int +first_name string PREHOOK: query: ALTER TABLE doctors ADD COLUMNS (last_name string) PREHOOK: type: ALTERTABLE_ADDCOLS PREHOOK: Input: default@doctors @@ -40,9 +40,9 @@ PREHOOK: Input: default@doctors POSTHOOK: query: DESCRIBE doctors POSTHOOK: type: DESCTABLE POSTHOOK: Input: default@doctors -number int from deserializer -first_name string from deserializer -last_name string from deserializer +number int +first_name string +last_name string PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/doctors.avro' INTO TABLE doctors PREHOOK: type: LOAD #### A masked pattern was here #### diff --git a/ql/src/test/results/clientpositive/avro_add_column2.q.out b/ql/src/test/results/clientpositive/avro_add_column2.q.out index 0e26cd59f3969898f0cd2d47ae4d271fe39824f3..630ae22e262d61d387393c0ff3c2003569d44bb9 100644 --- a/ql/src/test/results/clientpositive/avro_add_column2.q.out +++ b/ql/src/test/results/clientpositive/avro_add_column2.q.out @@ -50,8 +50,8 @@ POSTHOOK: query: INSERT INTO TABLE doctors_copy SELECT number, first_name FROM d POSTHOOK: type: QUERY POSTHOOK: Input: default@doctors POSTHOOK: Output: default@doctors_copy -POSTHOOK: Lineage: doctors_copy.first_name SIMPLE [(doctors)doctors.FieldSchema(name:first_name, type:string, comment:from deserializer), ] -POSTHOOK: Lineage: doctors_copy.number SIMPLE [(doctors)doctors.FieldSchema(name:number, type:int, comment:from deserializer), ] +POSTHOOK: Lineage: doctors_copy.first_name SIMPLE [(doctors)doctors.FieldSchema(name:first_name, type:string, comment:), ] +POSTHOOK: Lineage: doctors_copy.number SIMPLE [(doctors)doctors.FieldSchema(name:number, type:int, comment:), ] PREHOOK: query: ALTER TABLE doctors_copy ADD COLUMNS (last_name string) PREHOOK: type: ALTERTABLE_ADDCOLS PREHOOK: Input: default@doctors_copy @@ -74,9 +74,9 @@ PREHOOK: Input: default@doctors_copy POSTHOOK: query: DESCRIBE doctors_copy POSTHOOK: type: DESCTABLE POSTHOOK: Input: default@doctors_copy -number int from deserializer -first_name string from deserializer -last_name string from deserializer +number int +first_name string +last_name string PREHOOK: query: SELECT * FROM doctors_copy PREHOOK: type: QUERY PREHOOK: Input: default@doctors_copy diff --git a/ql/src/test/results/clientpositive/avro_add_column3.q.out b/ql/src/test/results/clientpositive/avro_add_column3.q.out index 0418b29ca95e71dca0b46bc3bd26bb1564192d1e..ced2d5cf52c3184b20c3e5cf0abc42e62d938712 100644 --- a/ql/src/test/results/clientpositive/avro_add_column3.q.out +++ b/ql/src/test/results/clientpositive/avro_add_column3.q.out @@ -52,8 +52,8 @@ POSTHOOK: query: INSERT INTO TABLE doctors_copy PARTITION(part=1) SELECT number, POSTHOOK: type: QUERY POSTHOOK: Input: default@doctors POSTHOOK: Output: default@doctors_copy@part=1 -POSTHOOK: Lineage: doctors_copy PARTITION(part=1).first_name SIMPLE [(doctors)doctors.FieldSchema(name:first_name, type:string, comment:from deserializer), ] -POSTHOOK: Lineage: doctors_copy PARTITION(part=1).number SIMPLE [(doctors)doctors.FieldSchema(name:number, type:int, comment:from deserializer), ] +POSTHOOK: Lineage: doctors_copy PARTITION(part=1).first_name SIMPLE [(doctors)doctors.FieldSchema(name:first_name, type:string, comment:), ] +POSTHOOK: Lineage: doctors_copy PARTITION(part=1).number SIMPLE [(doctors)doctors.FieldSchema(name:number, type:int, comment:), ] PREHOOK: query: ALTER TABLE doctors_copy ADD COLUMNS (last_name string) PREHOOK: type: ALTERTABLE_ADDCOLS PREHOOK: Input: default@doctors_copy @@ -68,9 +68,9 @@ PREHOOK: Input: default@doctors_copy POSTHOOK: query: DESCRIBE doctors_copy POSTHOOK: type: DESCTABLE POSTHOOK: Input: default@doctors_copy -number int from deserializer -first_name string from deserializer -last_name string from deserializer +number int +first_name string +last_name string part int # Partition Information diff --git a/ql/src/test/results/clientpositive/avro_change_schema.q.out b/ql/src/test/results/clientpositive/avro_change_schema.q.out index 5d52d215471527b00c5a277ddb77160249ea44a1..8c302d1cfaafb8ea0590f8f0592c414c2fcac41d 100644 --- a/ql/src/test/results/clientpositive/avro_change_schema.q.out +++ b/ql/src/test/results/clientpositive/avro_change_schema.q.out @@ -38,8 +38,8 @@ PREHOOK: Input: default@avro2 POSTHOOK: query: DESCRIBE avro2 POSTHOOK: type: DESCTABLE POSTHOOK: Input: default@avro2 -string1 string from deserializer -string2 string from deserializer +string1 string +string2 string PREHOOK: query: ALTER TABLE avro2 SET TBLPROPERTIES ('avro.schema.literal'='{ "namespace": "org.apache.hive", "name": "second_schema", "type": "record", @@ -68,6 +68,6 @@ PREHOOK: Input: default@avro2 POSTHOOK: query: DESCRIBE avro2 POSTHOOK: type: DESCTABLE POSTHOOK: Input: default@avro2 -int1 int from deserializer -float1 float from deserializer -double1 double from deserializer +int1 int +float1 float +double1 double diff --git a/ql/src/test/results/clientpositive/avro_comments.q.out b/ql/src/test/results/clientpositive/avro_comments.q.out new file mode 100644 index 0000000000000000000000000000000000000000..aab0e6a1c9732ae7f6dd78e61a0d8a70ec6b8e01 --- /dev/null +++ b/ql/src/test/results/clientpositive/avro_comments.q.out @@ -0,0 +1,398 @@ +PREHOOK: query: -- verify Avro columns comments +DROP TABLE IF EXISTS testAvroComments1 +PREHOOK: type: DROPTABLE +POSTHOOK: query: -- verify Avro columns comments +DROP TABLE IF EXISTS testAvroComments1 +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE testAvroComments1 +ROW FORMAT +SERDE 'org.apache.hadoop.hive.serde2.avro.AvroSerDe' +STORED AS +INPUTFORMAT 'org.apache.hadoop.hive.ql.io.avro.AvroContainerInputFormat' +OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.avro.AvroContainerOutputFormat' +TBLPROPERTIES ('avro.schema.literal'='{ + "namespace": "testing.hive.avro.serde", + "name": "doctors", + "type": "record", + "fields": [ + { + "name":"number", + "type":"int", + "doc":"Order of playing the role" + }, + { + "name":"first_name", + "type":"string", + "doc":"first name of actor playing role" + }, + { + "name":"last_name", + "type":"string", + "doc":"last name of actor playing role" + }, + { + "name":"extra_field", + "type":"string", + "doc":"an extra field not in the original file", + "default":"fishfingers and custard" + } + ] +}') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@testAvroComments1 +POSTHOOK: query: CREATE TABLE testAvroComments1 +ROW FORMAT +SERDE 'org.apache.hadoop.hive.serde2.avro.AvroSerDe' +STORED AS +INPUTFORMAT 'org.apache.hadoop.hive.ql.io.avro.AvroContainerInputFormat' +OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.avro.AvroContainerOutputFormat' +TBLPROPERTIES ('avro.schema.literal'='{ + "namespace": "testing.hive.avro.serde", + "name": "doctors", + "type": "record", + "fields": [ + { + "name":"number", + "type":"int", + "doc":"Order of playing the role" + }, + { + "name":"first_name", + "type":"string", + "doc":"first name of actor playing role" + }, + { + "name":"last_name", + "type":"string", + "doc":"last name of actor playing role" + }, + { + "name":"extra_field", + "type":"string", + "doc":"an extra field not in the original file", + "default":"fishfingers and custard" + } + ] +}') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@testAvroComments1 +PREHOOK: query: DESCRIBE testAvroComments1 +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@testavrocomments1 +POSTHOOK: query: DESCRIBE testAvroComments1 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@testavrocomments1 +number int Order of playing the role +first_name string first name of actor playing role +last_name string last name of actor playing role +extra_field string an extra field not in the original file +PREHOOK: query: DROP TABLE testAvroComments1 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@testavrocomments1 +PREHOOK: Output: default@testavrocomments1 +POSTHOOK: query: DROP TABLE testAvroComments1 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@testavrocomments1 +POSTHOOK: Output: default@testavrocomments1 +PREHOOK: query: DROP TABLE IF EXISTS testAvroComments2 +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE IF EXISTS testAvroComments2 +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE testAvroComments2 +ROW FORMAT +SERDE 'org.apache.hadoop.hive.serde2.avro.AvroSerDe' +STORED AS +INPUTFORMAT 'org.apache.hadoop.hive.ql.io.avro.AvroContainerInputFormat' +OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.avro.AvroContainerOutputFormat' +TBLPROPERTIES ('avro.schema.literal'='{ + "namespace": "testing.hive.avro.serde", + "name": "doctors", + "type": "record", + "fields": [ + { + "name":"number", + "type":"int", + "doc":"Order of playing the role" + }, + { + "name":"first_name", + "type":"string" + }, + { + "name":"last_name", + "type":"string", + "doc":"last name of actor playing role" + }, + { + "name":"extra_field", + "type":"string", + "default":"fishfingers and custard" + } + ] +}') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@testAvroComments2 +POSTHOOK: query: CREATE TABLE testAvroComments2 +ROW FORMAT +SERDE 'org.apache.hadoop.hive.serde2.avro.AvroSerDe' +STORED AS +INPUTFORMAT 'org.apache.hadoop.hive.ql.io.avro.AvroContainerInputFormat' +OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.avro.AvroContainerOutputFormat' +TBLPROPERTIES ('avro.schema.literal'='{ + "namespace": "testing.hive.avro.serde", + "name": "doctors", + "type": "record", + "fields": [ + { + "name":"number", + "type":"int", + "doc":"Order of playing the role" + }, + { + "name":"first_name", + "type":"string" + }, + { + "name":"last_name", + "type":"string", + "doc":"last name of actor playing role" + }, + { + "name":"extra_field", + "type":"string", + "default":"fishfingers and custard" + } + ] +}') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@testAvroComments2 +PREHOOK: query: DESCRIBE testAvroComments2 +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@testavrocomments2 +POSTHOOK: query: DESCRIBE testAvroComments2 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@testavrocomments2 +number int Order of playing the role +first_name string +last_name string last name of actor playing role +extra_field string +PREHOOK: query: DROP TABLE testAvroComments2 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@testavrocomments2 +PREHOOK: Output: default@testavrocomments2 +POSTHOOK: query: DROP TABLE testAvroComments2 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@testavrocomments2 +POSTHOOK: Output: default@testavrocomments2 +PREHOOK: query: DROP TABLE IF EXISTS testAvroComments3 +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE IF EXISTS testAvroComments3 +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE testAvroComments3 +ROW FORMAT +SERDE 'org.apache.hadoop.hive.serde2.avro.AvroSerDe' +STORED AS +INPUTFORMAT 'org.apache.hadoop.hive.ql.io.avro.AvroContainerInputFormat' +OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.avro.AvroContainerOutputFormat' +TBLPROPERTIES ('avro.schema.literal'='{ + "namespace": "testing.hive.avro.serde", + "name": "doctors", + "type": "record", + "fields": [ + { + "name":"number", + "type":"int" + }, + { + "name":"first_name", + "type":"string" + }, + { + "name":"last_name", + "type":"string" + }, + { + "name":"extra_field", + "type":"string", + "default":"fishfingers and custard" + } + ] +}') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@testAvroComments3 +POSTHOOK: query: CREATE TABLE testAvroComments3 +ROW FORMAT +SERDE 'org.apache.hadoop.hive.serde2.avro.AvroSerDe' +STORED AS +INPUTFORMAT 'org.apache.hadoop.hive.ql.io.avro.AvroContainerInputFormat' +OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.avro.AvroContainerOutputFormat' +TBLPROPERTIES ('avro.schema.literal'='{ + "namespace": "testing.hive.avro.serde", + "name": "doctors", + "type": "record", + "fields": [ + { + "name":"number", + "type":"int" + }, + { + "name":"first_name", + "type":"string" + }, + { + "name":"last_name", + "type":"string" + }, + { + "name":"extra_field", + "type":"string", + "default":"fishfingers and custard" + } + ] +}') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@testAvroComments3 +PREHOOK: query: DESCRIBE testAvroComments3 +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@testavrocomments3 +POSTHOOK: query: DESCRIBE testAvroComments3 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@testavrocomments3 +number int +first_name string +last_name string +extra_field string +PREHOOK: query: DROP TABLE testAvroComments3 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@testavrocomments3 +PREHOOK: Output: default@testavrocomments3 +POSTHOOK: query: DROP TABLE testAvroComments3 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@testavrocomments3 +POSTHOOK: Output: default@testavrocomments3 +PREHOOK: query: DROP TABLE IF EXISTS testAvroComments4 +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE IF EXISTS testAvroComments4 +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE testAvroComments4 ( + number int COMMENT "Order of playing the role", + first_name string COMMENT "first name of actor playing role", + last_name string COMMENT "last name of actor playing role", + extra_field string COMMENT "an extra field not in the original file") +STORED AS AVRO +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@testAvroComments4 +POSTHOOK: query: CREATE TABLE testAvroComments4 ( + number int COMMENT "Order of playing the role", + first_name string COMMENT "first name of actor playing role", + last_name string COMMENT "last name of actor playing role", + extra_field string COMMENT "an extra field not in the original file") +STORED AS AVRO +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@testAvroComments4 +PREHOOK: query: DESCRIBE testAvroComments4 +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@testavrocomments4 +POSTHOOK: query: DESCRIBE testAvroComments4 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@testavrocomments4 +number int Order of playing the role +first_name string first name of actor playing role +last_name string last name of actor playing role +extra_field string an extra field not in the original file +PREHOOK: query: DROP TABLE testAvroComments4 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@testavrocomments4 +PREHOOK: Output: default@testavrocomments4 +POSTHOOK: query: DROP TABLE testAvroComments4 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@testavrocomments4 +POSTHOOK: Output: default@testavrocomments4 +PREHOOK: query: DROP TABLE IF EXISTS testAvroComments5 +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE IF EXISTS testAvroComments5 +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE testAvroComments5 ( + number int COMMENT "Order of playing the role", + first_name string, + last_name string COMMENT "last name of actor playing role", + extra_field string) +STORED AS AVRO +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@testAvroComments5 +POSTHOOK: query: CREATE TABLE testAvroComments5 ( + number int COMMENT "Order of playing the role", + first_name string, + last_name string COMMENT "last name of actor playing role", + extra_field string) +STORED AS AVRO +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@testAvroComments5 +PREHOOK: query: DESCRIBE testAvroComments5 +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@testavrocomments5 +POSTHOOK: query: DESCRIBE testAvroComments5 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@testavrocomments5 +number int Order of playing the role +first_name string +last_name string last name of actor playing role +extra_field string +PREHOOK: query: DROP TABLE testAvroComments5 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@testavrocomments5 +PREHOOK: Output: default@testavrocomments5 +POSTHOOK: query: DROP TABLE testAvroComments5 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@testavrocomments5 +POSTHOOK: Output: default@testavrocomments5 +PREHOOK: query: DROP TABLE IF EXISTS testAvroComments6 +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE IF EXISTS testAvroComments6 +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE testAvroComments6 ( + number int, + first_name string, + last_name string, + extra_field string) +STORED AS AVRO +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@testAvroComments6 +POSTHOOK: query: CREATE TABLE testAvroComments6 ( + number int, + first_name string, + last_name string, + extra_field string) +STORED AS AVRO +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@testAvroComments6 +PREHOOK: query: DESCRIBE testAvroComments6 +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@testavrocomments6 +POSTHOOK: query: DESCRIBE testAvroComments6 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@testavrocomments6 +number int +first_name string +last_name string +extra_field string +PREHOOK: query: DROP TABLE testAvroComments6 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@testavrocomments6 +PREHOOK: Output: default@testavrocomments6 +POSTHOOK: query: DROP TABLE testAvroComments6 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@testavrocomments6 +POSTHOOK: Output: default@testavrocomments6 diff --git a/ql/src/test/results/clientpositive/avro_compression_enabled.q.out b/ql/src/test/results/clientpositive/avro_compression_enabled.q.out index bcb967a11516b33eac045d160f88946606ce75f7..882010753beb0f3dc52faf3516046135bb752be2 100644 --- a/ql/src/test/results/clientpositive/avro_compression_enabled.q.out +++ b/ql/src/test/results/clientpositive/avro_compression_enabled.q.out @@ -29,7 +29,7 @@ TBLPROPERTIES ('avro.schema.literal'='{ { "name":"extra_field", "type":"string", - "doc:":"an extra field not in the original file", + "doc":"an extra field not in the original file", "default":"fishfingers and custard" } ] @@ -68,7 +68,7 @@ TBLPROPERTIES ('avro.schema.literal'='{ { "name":"extra_field", "type":"string", - "doc:":"an extra field not in the original file", + "doc":"an extra field not in the original file", "default":"fishfingers and custard" } ] diff --git a/ql/src/test/results/clientpositive/avro_decimal.q.out b/ql/src/test/results/clientpositive/avro_decimal.q.out index 88268ceb3a5dddc50d48b502dba17647f8d8e5fe..7ba376ea913d487bf55fd69d85ec1209b64a4695 100644 --- a/ql/src/test/results/clientpositive/avro_decimal.q.out +++ b/ql/src/test/results/clientpositive/avro_decimal.q.out @@ -79,8 +79,8 @@ PREHOOK: Input: default@avro_dec POSTHOOK: query: DESC avro_dec POSTHOOK: type: DESCTABLE POSTHOOK: Input: default@avro_dec -name string from deserializer -value decimal(5,2) from deserializer +name string +value decimal(5,2) PREHOOK: query: INSERT OVERWRITE TABLE avro_dec select name, value from dec PREHOOK: type: QUERY PREHOOK: Input: default@dec @@ -153,8 +153,8 @@ PREHOOK: Input: default@avro_dec1 POSTHOOK: query: DESC avro_dec1 POSTHOOK: type: DESCTABLE POSTHOOK: Input: default@avro_dec1 -name string from deserializer -value decimal(4,1) from deserializer +name string +value decimal(4,1) PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/dec.avro' into TABLE avro_dec1 PREHOOK: type: LOAD #### A masked pattern was here #### diff --git a/ql/src/test/results/clientpositive/avro_decimal_native.q.out b/ql/src/test/results/clientpositive/avro_decimal_native.q.out index c8ae0fb2404c8891c4e307d82d1c2d7f180eae51..318be3da66a754de0414c2f63c2a831b33a1eba7 100644 --- a/ql/src/test/results/clientpositive/avro_decimal_native.q.out +++ b/ql/src/test/results/clientpositive/avro_decimal_native.q.out @@ -65,8 +65,8 @@ PREHOOK: Input: default@avro_dec POSTHOOK: query: DESC avro_dec POSTHOOK: type: DESCTABLE POSTHOOK: Input: default@avro_dec -name string from deserializer -value decimal(5,2) from deserializer +name string +value decimal(5,2) PREHOOK: query: INSERT OVERWRITE TABLE avro_dec SELECT name, value FROM dec PREHOOK: type: QUERY PREHOOK: Input: default@dec @@ -121,8 +121,8 @@ PREHOOK: Input: default@avro_dec1 POSTHOOK: query: DESC avro_dec1 POSTHOOK: type: DESCTABLE POSTHOOK: Input: default@avro_dec1 -name string from deserializer -value decimal(4,1) from deserializer +name string +value decimal(4,1) PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/dec.avro' INTO TABLE avro_dec1 PREHOOK: type: LOAD #### A masked pattern was here #### diff --git a/ql/src/test/results/clientpositive/avro_evolved_schemas.q.out b/ql/src/test/results/clientpositive/avro_evolved_schemas.q.out index 2f81b1ab9a6a266318945135726de7d23c809d8c..e7771f6aa4ea9c83e992f9cab9426cf96b649f13 100644 --- a/ql/src/test/results/clientpositive/avro_evolved_schemas.q.out +++ b/ql/src/test/results/clientpositive/avro_evolved_schemas.q.out @@ -30,7 +30,7 @@ TBLPROPERTIES ('avro.schema.literal'='{ { "name":"extra_field", "type":"string", - "doc:":"an extra field not in the original file", + "doc":"an extra field not in the original file", "default":"fishfingers and custard" } ] @@ -70,7 +70,7 @@ TBLPROPERTIES ('avro.schema.literal'='{ { "name":"extra_field", "type":"string", - "doc:":"an extra field not in the original file", + "doc":"an extra field not in the original file", "default":"fishfingers and custard" } ] @@ -84,10 +84,10 @@ PREHOOK: Input: default@doctors_with_new_field POSTHOOK: query: DESCRIBE doctors_with_new_field POSTHOOK: type: DESCTABLE POSTHOOK: Input: default@doctors_with_new_field -number int from deserializer -first_name string from deserializer -last_name string from deserializer -extra_field string from deserializer +number int Order of playing the role +first_name string first name of actor playing role +last_name string last name of actor playing role +extra_field string an extra field not in the original file PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/doctors.avro' INTO TABLE doctors_with_new_field PREHOOK: type: LOAD #### A masked pattern was here #### diff --git a/ql/src/test/results/clientpositive/avro_joins.q.out b/ql/src/test/results/clientpositive/avro_joins.q.out index 7e47fe0f25ad9b90971c26fc1fee95ceb32c0862..b9f233bb4d7619ceb3319278ef1a6a7446431f8a 100644 --- a/ql/src/test/results/clientpositive/avro_joins.q.out +++ b/ql/src/test/results/clientpositive/avro_joins.q.out @@ -31,7 +31,7 @@ TBLPROPERTIES ('avro.schema.literal'='{ { "name":"extra_field", "type":"string", - "doc:":"an extra field not in the original file", + "doc":"an extra field not in the original file", "default":"fishfingers and custard" } ] @@ -72,7 +72,7 @@ TBLPROPERTIES ('avro.schema.literal'='{ { "name":"extra_field", "type":"string", - "doc:":"an extra field not in the original file", + "doc":"an extra field not in the original file", "default":"fishfingers and custard" } ] @@ -86,10 +86,10 @@ PREHOOK: Input: default@doctors4 POSTHOOK: query: DESCRIBE doctors4 POSTHOOK: type: DESCTABLE POSTHOOK: Input: default@doctors4 -number int from deserializer -first_name string from deserializer -last_name string from deserializer -extra_field string from deserializer +number int Order of playing the role +first_name string first name of actor playing role +last_name string last name of actor playing role +extra_field string an extra field not in the original file PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/doctors.avro' INTO TABLE doctors4 PREHOOK: type: LOAD #### A masked pattern was here #### @@ -166,9 +166,9 @@ PREHOOK: Input: default@episodes POSTHOOK: query: DESCRIBE episodes POSTHOOK: type: DESCTABLE POSTHOOK: Input: default@episodes -title string from deserializer -air_date string from deserializer -doctor int from deserializer +title string episode title +air_date string initial date +doctor int main actor playing the Doctor in episode PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/episodes.avro' INTO TABLE episodes PREHOOK: type: LOAD #### A masked pattern was here #### diff --git a/ql/src/test/results/clientpositive/avro_joins_native.q.out b/ql/src/test/results/clientpositive/avro_joins_native.q.out index ca22a7279bc285600ffa40f76707fdd7e94dbb23..09a15498c8cf969d695289b8461bf57cc2e7dd17 100644 --- a/ql/src/test/results/clientpositive/avro_joins_native.q.out +++ b/ql/src/test/results/clientpositive/avro_joins_native.q.out @@ -28,9 +28,9 @@ PREHOOK: Input: default@doctors4 POSTHOOK: query: DESCRIBE doctors4 POSTHOOK: type: DESCTABLE POSTHOOK: Input: default@doctors4 -number int from deserializer -first_name string from deserializer -last_name string from deserializer +number int Order of playing the role +first_name string first name of actor playing role +last_name string last name of actor playing role PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/doctors.avro' INTO TABLE doctors4 PREHOOK: type: LOAD #### A masked pattern was here #### @@ -61,9 +61,9 @@ PREHOOK: Input: default@episodes POSTHOOK: query: DESCRIBE episodes POSTHOOK: type: DESCTABLE POSTHOOK: Input: default@episodes -title string from deserializer -air_date string from deserializer -doctor int from deserializer +title string episode title +air_date string initial date +doctor int main actor playing the Doctor in episode PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/episodes.avro' INTO TABLE episodes PREHOOK: type: LOAD #### A masked pattern was here #### diff --git a/ql/src/test/results/clientpositive/avro_native.q.out b/ql/src/test/results/clientpositive/avro_native.q.out index 2de4df1b8452e0b091a8ce3b6f5a9369ff208290..03e64c49d577a354231f50f052072ba1dee355cb 100644 --- a/ql/src/test/results/clientpositive/avro_native.q.out +++ b/ql/src/test/results/clientpositive/avro_native.q.out @@ -26,9 +26,9 @@ PREHOOK: Input: default@doctors POSTHOOK: query: DESCRIBE doctors POSTHOOK: type: DESCTABLE POSTHOOK: Input: default@doctors -number int from deserializer -first_name string from deserializer -last_name string from deserializer +number int +first_name string +last_name string PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/doctors.avro' INTO TABLE doctors PREHOOK: type: LOAD #### A masked pattern was here #### diff --git a/ql/src/test/results/clientpositive/avro_partitioned.q.out b/ql/src/test/results/clientpositive/avro_partitioned.q.out index 890cd9f764e35f9d99b42a7083f0bb49e2ec7bab..c642736e3a83ae24dcc4190d5a93d598a34f2621 100644 --- a/ql/src/test/results/clientpositive/avro_partitioned.q.out +++ b/ql/src/test/results/clientpositive/avro_partitioned.q.out @@ -150,27 +150,27 @@ POSTHOOK: Output: default@episodes_partitioned@doctor_pt=4 POSTHOOK: Output: default@episodes_partitioned@doctor_pt=5 POSTHOOK: Output: default@episodes_partitioned@doctor_pt=6 POSTHOOK: Output: default@episodes_partitioned@doctor_pt=9 -POSTHOOK: Lineage: episodes_partitioned PARTITION(doctor_pt=11).air_date SIMPLE [(episodes)episodes.FieldSchema(name:air_date, type:string, comment:from deserializer), ] -POSTHOOK: Lineage: episodes_partitioned PARTITION(doctor_pt=11).doctor SIMPLE [(episodes)episodes.FieldSchema(name:doctor, type:int, comment:from deserializer), ] -POSTHOOK: Lineage: episodes_partitioned PARTITION(doctor_pt=11).title SIMPLE [(episodes)episodes.FieldSchema(name:title, type:string, comment:from deserializer), ] -POSTHOOK: Lineage: episodes_partitioned PARTITION(doctor_pt=1).air_date SIMPLE [(episodes)episodes.FieldSchema(name:air_date, type:string, comment:from deserializer), ] -POSTHOOK: Lineage: episodes_partitioned PARTITION(doctor_pt=1).doctor SIMPLE [(episodes)episodes.FieldSchema(name:doctor, type:int, comment:from deserializer), ] -POSTHOOK: Lineage: episodes_partitioned PARTITION(doctor_pt=1).title SIMPLE [(episodes)episodes.FieldSchema(name:title, type:string, comment:from deserializer), ] -POSTHOOK: Lineage: episodes_partitioned PARTITION(doctor_pt=2).air_date SIMPLE [(episodes)episodes.FieldSchema(name:air_date, type:string, comment:from deserializer), ] -POSTHOOK: Lineage: episodes_partitioned PARTITION(doctor_pt=2).doctor SIMPLE [(episodes)episodes.FieldSchema(name:doctor, type:int, comment:from deserializer), ] -POSTHOOK: Lineage: episodes_partitioned PARTITION(doctor_pt=2).title SIMPLE [(episodes)episodes.FieldSchema(name:title, type:string, comment:from deserializer), ] -POSTHOOK: Lineage: episodes_partitioned PARTITION(doctor_pt=4).air_date SIMPLE [(episodes)episodes.FieldSchema(name:air_date, type:string, comment:from deserializer), ] -POSTHOOK: Lineage: episodes_partitioned PARTITION(doctor_pt=4).doctor SIMPLE [(episodes)episodes.FieldSchema(name:doctor, type:int, comment:from deserializer), ] -POSTHOOK: Lineage: episodes_partitioned PARTITION(doctor_pt=4).title SIMPLE [(episodes)episodes.FieldSchema(name:title, type:string, comment:from deserializer), ] -POSTHOOK: Lineage: episodes_partitioned PARTITION(doctor_pt=5).air_date SIMPLE [(episodes)episodes.FieldSchema(name:air_date, type:string, comment:from deserializer), ] -POSTHOOK: Lineage: episodes_partitioned PARTITION(doctor_pt=5).doctor SIMPLE [(episodes)episodes.FieldSchema(name:doctor, type:int, comment:from deserializer), ] -POSTHOOK: Lineage: episodes_partitioned PARTITION(doctor_pt=5).title SIMPLE [(episodes)episodes.FieldSchema(name:title, type:string, comment:from deserializer), ] -POSTHOOK: Lineage: episodes_partitioned PARTITION(doctor_pt=6).air_date SIMPLE [(episodes)episodes.FieldSchema(name:air_date, type:string, comment:from deserializer), ] -POSTHOOK: Lineage: episodes_partitioned PARTITION(doctor_pt=6).doctor SIMPLE [(episodes)episodes.FieldSchema(name:doctor, type:int, comment:from deserializer), ] -POSTHOOK: Lineage: episodes_partitioned PARTITION(doctor_pt=6).title SIMPLE [(episodes)episodes.FieldSchema(name:title, type:string, comment:from deserializer), ] -POSTHOOK: Lineage: episodes_partitioned PARTITION(doctor_pt=9).air_date SIMPLE [(episodes)episodes.FieldSchema(name:air_date, type:string, comment:from deserializer), ] -POSTHOOK: Lineage: episodes_partitioned PARTITION(doctor_pt=9).doctor SIMPLE [(episodes)episodes.FieldSchema(name:doctor, type:int, comment:from deserializer), ] -POSTHOOK: Lineage: episodes_partitioned PARTITION(doctor_pt=9).title SIMPLE [(episodes)episodes.FieldSchema(name:title, type:string, comment:from deserializer), ] +POSTHOOK: Lineage: episodes_partitioned PARTITION(doctor_pt=11).air_date SIMPLE [(episodes)episodes.FieldSchema(name:air_date, type:string, comment:initial date), ] +POSTHOOK: Lineage: episodes_partitioned PARTITION(doctor_pt=11).doctor SIMPLE [(episodes)episodes.FieldSchema(name:doctor, type:int, comment:main actor playing the Doctor in episode), ] +POSTHOOK: Lineage: episodes_partitioned PARTITION(doctor_pt=11).title SIMPLE [(episodes)episodes.FieldSchema(name:title, type:string, comment:episode title), ] +POSTHOOK: Lineage: episodes_partitioned PARTITION(doctor_pt=1).air_date SIMPLE [(episodes)episodes.FieldSchema(name:air_date, type:string, comment:initial date), ] +POSTHOOK: Lineage: episodes_partitioned PARTITION(doctor_pt=1).doctor SIMPLE [(episodes)episodes.FieldSchema(name:doctor, type:int, comment:main actor playing the Doctor in episode), ] +POSTHOOK: Lineage: episodes_partitioned PARTITION(doctor_pt=1).title SIMPLE [(episodes)episodes.FieldSchema(name:title, type:string, comment:episode title), ] +POSTHOOK: Lineage: episodes_partitioned PARTITION(doctor_pt=2).air_date SIMPLE [(episodes)episodes.FieldSchema(name:air_date, type:string, comment:initial date), ] +POSTHOOK: Lineage: episodes_partitioned PARTITION(doctor_pt=2).doctor SIMPLE [(episodes)episodes.FieldSchema(name:doctor, type:int, comment:main actor playing the Doctor in episode), ] +POSTHOOK: Lineage: episodes_partitioned PARTITION(doctor_pt=2).title SIMPLE [(episodes)episodes.FieldSchema(name:title, type:string, comment:episode title), ] +POSTHOOK: Lineage: episodes_partitioned PARTITION(doctor_pt=4).air_date SIMPLE [(episodes)episodes.FieldSchema(name:air_date, type:string, comment:initial date), ] +POSTHOOK: Lineage: episodes_partitioned PARTITION(doctor_pt=4).doctor SIMPLE [(episodes)episodes.FieldSchema(name:doctor, type:int, comment:main actor playing the Doctor in episode), ] +POSTHOOK: Lineage: episodes_partitioned PARTITION(doctor_pt=4).title SIMPLE [(episodes)episodes.FieldSchema(name:title, type:string, comment:episode title), ] +POSTHOOK: Lineage: episodes_partitioned PARTITION(doctor_pt=5).air_date SIMPLE [(episodes)episodes.FieldSchema(name:air_date, type:string, comment:initial date), ] +POSTHOOK: Lineage: episodes_partitioned PARTITION(doctor_pt=5).doctor SIMPLE [(episodes)episodes.FieldSchema(name:doctor, type:int, comment:main actor playing the Doctor in episode), ] +POSTHOOK: Lineage: episodes_partitioned PARTITION(doctor_pt=5).title SIMPLE [(episodes)episodes.FieldSchema(name:title, type:string, comment:episode title), ] +POSTHOOK: Lineage: episodes_partitioned PARTITION(doctor_pt=6).air_date SIMPLE [(episodes)episodes.FieldSchema(name:air_date, type:string, comment:initial date), ] +POSTHOOK: Lineage: episodes_partitioned PARTITION(doctor_pt=6).doctor SIMPLE [(episodes)episodes.FieldSchema(name:doctor, type:int, comment:main actor playing the Doctor in episode), ] +POSTHOOK: Lineage: episodes_partitioned PARTITION(doctor_pt=6).title SIMPLE [(episodes)episodes.FieldSchema(name:title, type:string, comment:episode title), ] +POSTHOOK: Lineage: episodes_partitioned PARTITION(doctor_pt=9).air_date SIMPLE [(episodes)episodes.FieldSchema(name:air_date, type:string, comment:initial date), ] +POSTHOOK: Lineage: episodes_partitioned PARTITION(doctor_pt=9).doctor SIMPLE [(episodes)episodes.FieldSchema(name:doctor, type:int, comment:main actor playing the Doctor in episode), ] +POSTHOOK: Lineage: episodes_partitioned PARTITION(doctor_pt=9).title SIMPLE [(episodes)episodes.FieldSchema(name:title, type:string, comment:episode title), ] PREHOOK: query: SELECT * FROM episodes_partitioned WHERE doctor_pt > 6 PREHOOK: type: QUERY PREHOOK: Input: default@episodes_partitioned @@ -360,27 +360,27 @@ POSTHOOK: Output: default@episodes_partitioned_serdeproperties@doctor_pt=4 POSTHOOK: Output: default@episodes_partitioned_serdeproperties@doctor_pt=5 POSTHOOK: Output: default@episodes_partitioned_serdeproperties@doctor_pt=6 POSTHOOK: Output: default@episodes_partitioned_serdeproperties@doctor_pt=9 -POSTHOOK: Lineage: episodes_partitioned_serdeproperties PARTITION(doctor_pt=11).air_date SIMPLE [(episodes)episodes.FieldSchema(name:air_date, type:string, comment:from deserializer), ] -POSTHOOK: Lineage: episodes_partitioned_serdeproperties PARTITION(doctor_pt=11).doctor SIMPLE [(episodes)episodes.FieldSchema(name:doctor, type:int, comment:from deserializer), ] -POSTHOOK: Lineage: episodes_partitioned_serdeproperties PARTITION(doctor_pt=11).title SIMPLE [(episodes)episodes.FieldSchema(name:title, type:string, comment:from deserializer), ] -POSTHOOK: Lineage: episodes_partitioned_serdeproperties PARTITION(doctor_pt=1).air_date SIMPLE [(episodes)episodes.FieldSchema(name:air_date, type:string, comment:from deserializer), ] -POSTHOOK: Lineage: episodes_partitioned_serdeproperties PARTITION(doctor_pt=1).doctor SIMPLE [(episodes)episodes.FieldSchema(name:doctor, type:int, comment:from deserializer), ] -POSTHOOK: Lineage: episodes_partitioned_serdeproperties PARTITION(doctor_pt=1).title SIMPLE [(episodes)episodes.FieldSchema(name:title, type:string, comment:from deserializer), ] -POSTHOOK: Lineage: episodes_partitioned_serdeproperties PARTITION(doctor_pt=2).air_date SIMPLE [(episodes)episodes.FieldSchema(name:air_date, type:string, comment:from deserializer), ] -POSTHOOK: Lineage: episodes_partitioned_serdeproperties PARTITION(doctor_pt=2).doctor SIMPLE [(episodes)episodes.FieldSchema(name:doctor, type:int, comment:from deserializer), ] -POSTHOOK: Lineage: episodes_partitioned_serdeproperties PARTITION(doctor_pt=2).title SIMPLE [(episodes)episodes.FieldSchema(name:title, type:string, comment:from deserializer), ] -POSTHOOK: Lineage: episodes_partitioned_serdeproperties PARTITION(doctor_pt=4).air_date SIMPLE [(episodes)episodes.FieldSchema(name:air_date, type:string, comment:from deserializer), ] -POSTHOOK: Lineage: episodes_partitioned_serdeproperties PARTITION(doctor_pt=4).doctor SIMPLE [(episodes)episodes.FieldSchema(name:doctor, type:int, comment:from deserializer), ] -POSTHOOK: Lineage: episodes_partitioned_serdeproperties PARTITION(doctor_pt=4).title SIMPLE [(episodes)episodes.FieldSchema(name:title, type:string, comment:from deserializer), ] -POSTHOOK: Lineage: episodes_partitioned_serdeproperties PARTITION(doctor_pt=5).air_date SIMPLE [(episodes)episodes.FieldSchema(name:air_date, type:string, comment:from deserializer), ] -POSTHOOK: Lineage: episodes_partitioned_serdeproperties PARTITION(doctor_pt=5).doctor SIMPLE [(episodes)episodes.FieldSchema(name:doctor, type:int, comment:from deserializer), ] -POSTHOOK: Lineage: episodes_partitioned_serdeproperties PARTITION(doctor_pt=5).title SIMPLE [(episodes)episodes.FieldSchema(name:title, type:string, comment:from deserializer), ] -POSTHOOK: Lineage: episodes_partitioned_serdeproperties PARTITION(doctor_pt=6).air_date SIMPLE [(episodes)episodes.FieldSchema(name:air_date, type:string, comment:from deserializer), ] -POSTHOOK: Lineage: episodes_partitioned_serdeproperties PARTITION(doctor_pt=6).doctor SIMPLE [(episodes)episodes.FieldSchema(name:doctor, type:int, comment:from deserializer), ] -POSTHOOK: Lineage: episodes_partitioned_serdeproperties PARTITION(doctor_pt=6).title SIMPLE [(episodes)episodes.FieldSchema(name:title, type:string, comment:from deserializer), ] -POSTHOOK: Lineage: episodes_partitioned_serdeproperties PARTITION(doctor_pt=9).air_date SIMPLE [(episodes)episodes.FieldSchema(name:air_date, type:string, comment:from deserializer), ] -POSTHOOK: Lineage: episodes_partitioned_serdeproperties PARTITION(doctor_pt=9).doctor SIMPLE [(episodes)episodes.FieldSchema(name:doctor, type:int, comment:from deserializer), ] -POSTHOOK: Lineage: episodes_partitioned_serdeproperties PARTITION(doctor_pt=9).title SIMPLE [(episodes)episodes.FieldSchema(name:title, type:string, comment:from deserializer), ] +POSTHOOK: Lineage: episodes_partitioned_serdeproperties PARTITION(doctor_pt=11).air_date SIMPLE [(episodes)episodes.FieldSchema(name:air_date, type:string, comment:initial date), ] +POSTHOOK: Lineage: episodes_partitioned_serdeproperties PARTITION(doctor_pt=11).doctor SIMPLE [(episodes)episodes.FieldSchema(name:doctor, type:int, comment:main actor playing the Doctor in episode), ] +POSTHOOK: Lineage: episodes_partitioned_serdeproperties PARTITION(doctor_pt=11).title SIMPLE [(episodes)episodes.FieldSchema(name:title, type:string, comment:episode title), ] +POSTHOOK: Lineage: episodes_partitioned_serdeproperties PARTITION(doctor_pt=1).air_date SIMPLE [(episodes)episodes.FieldSchema(name:air_date, type:string, comment:initial date), ] +POSTHOOK: Lineage: episodes_partitioned_serdeproperties PARTITION(doctor_pt=1).doctor SIMPLE [(episodes)episodes.FieldSchema(name:doctor, type:int, comment:main actor playing the Doctor in episode), ] +POSTHOOK: Lineage: episodes_partitioned_serdeproperties PARTITION(doctor_pt=1).title SIMPLE [(episodes)episodes.FieldSchema(name:title, type:string, comment:episode title), ] +POSTHOOK: Lineage: episodes_partitioned_serdeproperties PARTITION(doctor_pt=2).air_date SIMPLE [(episodes)episodes.FieldSchema(name:air_date, type:string, comment:initial date), ] +POSTHOOK: Lineage: episodes_partitioned_serdeproperties PARTITION(doctor_pt=2).doctor SIMPLE [(episodes)episodes.FieldSchema(name:doctor, type:int, comment:main actor playing the Doctor in episode), ] +POSTHOOK: Lineage: episodes_partitioned_serdeproperties PARTITION(doctor_pt=2).title SIMPLE [(episodes)episodes.FieldSchema(name:title, type:string, comment:episode title), ] +POSTHOOK: Lineage: episodes_partitioned_serdeproperties PARTITION(doctor_pt=4).air_date SIMPLE [(episodes)episodes.FieldSchema(name:air_date, type:string, comment:initial date), ] +POSTHOOK: Lineage: episodes_partitioned_serdeproperties PARTITION(doctor_pt=4).doctor SIMPLE [(episodes)episodes.FieldSchema(name:doctor, type:int, comment:main actor playing the Doctor in episode), ] +POSTHOOK: Lineage: episodes_partitioned_serdeproperties PARTITION(doctor_pt=4).title SIMPLE [(episodes)episodes.FieldSchema(name:title, type:string, comment:episode title), ] +POSTHOOK: Lineage: episodes_partitioned_serdeproperties PARTITION(doctor_pt=5).air_date SIMPLE [(episodes)episodes.FieldSchema(name:air_date, type:string, comment:initial date), ] +POSTHOOK: Lineage: episodes_partitioned_serdeproperties PARTITION(doctor_pt=5).doctor SIMPLE [(episodes)episodes.FieldSchema(name:doctor, type:int, comment:main actor playing the Doctor in episode), ] +POSTHOOK: Lineage: episodes_partitioned_serdeproperties PARTITION(doctor_pt=5).title SIMPLE [(episodes)episodes.FieldSchema(name:title, type:string, comment:episode title), ] +POSTHOOK: Lineage: episodes_partitioned_serdeproperties PARTITION(doctor_pt=6).air_date SIMPLE [(episodes)episodes.FieldSchema(name:air_date, type:string, comment:initial date), ] +POSTHOOK: Lineage: episodes_partitioned_serdeproperties PARTITION(doctor_pt=6).doctor SIMPLE [(episodes)episodes.FieldSchema(name:doctor, type:int, comment:main actor playing the Doctor in episode), ] +POSTHOOK: Lineage: episodes_partitioned_serdeproperties PARTITION(doctor_pt=6).title SIMPLE [(episodes)episodes.FieldSchema(name:title, type:string, comment:episode title), ] +POSTHOOK: Lineage: episodes_partitioned_serdeproperties PARTITION(doctor_pt=9).air_date SIMPLE [(episodes)episodes.FieldSchema(name:air_date, type:string, comment:initial date), ] +POSTHOOK: Lineage: episodes_partitioned_serdeproperties PARTITION(doctor_pt=9).doctor SIMPLE [(episodes)episodes.FieldSchema(name:doctor, type:int, comment:main actor playing the Doctor in episode), ] +POSTHOOK: Lineage: episodes_partitioned_serdeproperties PARTITION(doctor_pt=9).title SIMPLE [(episodes)episodes.FieldSchema(name:title, type:string, comment:episode title), ] PREHOOK: query: -- Evolve the table schema by adding new array field "cast_and_crew" ALTER TABLE episodes_partitioned_serdeproperties SET SERDE 'org.apache.hadoop.hive.serde2.avro.AvroSerDe' diff --git a/ql/src/test/results/clientpositive/avro_partitioned_native.q.out b/ql/src/test/results/clientpositive/avro_partitioned_native.q.out index c02c656a45d54065280c4a30c0179f15d9a49267..ea11b4cbc5149a8c5d6ead8460799aa506d70eec 100644 --- a/ql/src/test/results/clientpositive/avro_partitioned_native.q.out +++ b/ql/src/test/results/clientpositive/avro_partitioned_native.q.out @@ -60,27 +60,27 @@ POSTHOOK: Output: default@episodes_partitioned@doctor_pt=4 POSTHOOK: Output: default@episodes_partitioned@doctor_pt=5 POSTHOOK: Output: default@episodes_partitioned@doctor_pt=6 POSTHOOK: Output: default@episodes_partitioned@doctor_pt=9 -POSTHOOK: Lineage: episodes_partitioned PARTITION(doctor_pt=11).air_date SIMPLE [(episodes)episodes.FieldSchema(name:air_date, type:string, comment:from deserializer), ] -POSTHOOK: Lineage: episodes_partitioned PARTITION(doctor_pt=11).doctor SIMPLE [(episodes)episodes.FieldSchema(name:doctor, type:int, comment:from deserializer), ] -POSTHOOK: Lineage: episodes_partitioned PARTITION(doctor_pt=11).title SIMPLE [(episodes)episodes.FieldSchema(name:title, type:string, comment:from deserializer), ] -POSTHOOK: Lineage: episodes_partitioned PARTITION(doctor_pt=1).air_date SIMPLE [(episodes)episodes.FieldSchema(name:air_date, type:string, comment:from deserializer), ] -POSTHOOK: Lineage: episodes_partitioned PARTITION(doctor_pt=1).doctor SIMPLE [(episodes)episodes.FieldSchema(name:doctor, type:int, comment:from deserializer), ] -POSTHOOK: Lineage: episodes_partitioned PARTITION(doctor_pt=1).title SIMPLE [(episodes)episodes.FieldSchema(name:title, type:string, comment:from deserializer), ] -POSTHOOK: Lineage: episodes_partitioned PARTITION(doctor_pt=2).air_date SIMPLE [(episodes)episodes.FieldSchema(name:air_date, type:string, comment:from deserializer), ] -POSTHOOK: Lineage: episodes_partitioned PARTITION(doctor_pt=2).doctor SIMPLE [(episodes)episodes.FieldSchema(name:doctor, type:int, comment:from deserializer), ] -POSTHOOK: Lineage: episodes_partitioned PARTITION(doctor_pt=2).title SIMPLE [(episodes)episodes.FieldSchema(name:title, type:string, comment:from deserializer), ] -POSTHOOK: Lineage: episodes_partitioned PARTITION(doctor_pt=4).air_date SIMPLE [(episodes)episodes.FieldSchema(name:air_date, type:string, comment:from deserializer), ] -POSTHOOK: Lineage: episodes_partitioned PARTITION(doctor_pt=4).doctor SIMPLE [(episodes)episodes.FieldSchema(name:doctor, type:int, comment:from deserializer), ] -POSTHOOK: Lineage: episodes_partitioned PARTITION(doctor_pt=4).title SIMPLE [(episodes)episodes.FieldSchema(name:title, type:string, comment:from deserializer), ] -POSTHOOK: Lineage: episodes_partitioned PARTITION(doctor_pt=5).air_date SIMPLE [(episodes)episodes.FieldSchema(name:air_date, type:string, comment:from deserializer), ] -POSTHOOK: Lineage: episodes_partitioned PARTITION(doctor_pt=5).doctor SIMPLE [(episodes)episodes.FieldSchema(name:doctor, type:int, comment:from deserializer), ] -POSTHOOK: Lineage: episodes_partitioned PARTITION(doctor_pt=5).title SIMPLE [(episodes)episodes.FieldSchema(name:title, type:string, comment:from deserializer), ] -POSTHOOK: Lineage: episodes_partitioned PARTITION(doctor_pt=6).air_date SIMPLE [(episodes)episodes.FieldSchema(name:air_date, type:string, comment:from deserializer), ] -POSTHOOK: Lineage: episodes_partitioned PARTITION(doctor_pt=6).doctor SIMPLE [(episodes)episodes.FieldSchema(name:doctor, type:int, comment:from deserializer), ] -POSTHOOK: Lineage: episodes_partitioned PARTITION(doctor_pt=6).title SIMPLE [(episodes)episodes.FieldSchema(name:title, type:string, comment:from deserializer), ] -POSTHOOK: Lineage: episodes_partitioned PARTITION(doctor_pt=9).air_date SIMPLE [(episodes)episodes.FieldSchema(name:air_date, type:string, comment:from deserializer), ] -POSTHOOK: Lineage: episodes_partitioned PARTITION(doctor_pt=9).doctor SIMPLE [(episodes)episodes.FieldSchema(name:doctor, type:int, comment:from deserializer), ] -POSTHOOK: Lineage: episodes_partitioned PARTITION(doctor_pt=9).title SIMPLE [(episodes)episodes.FieldSchema(name:title, type:string, comment:from deserializer), ] +POSTHOOK: Lineage: episodes_partitioned PARTITION(doctor_pt=11).air_date SIMPLE [(episodes)episodes.FieldSchema(name:air_date, type:string, comment:initial date), ] +POSTHOOK: Lineage: episodes_partitioned PARTITION(doctor_pt=11).doctor SIMPLE [(episodes)episodes.FieldSchema(name:doctor, type:int, comment:main actor playing the Doctor in episode), ] +POSTHOOK: Lineage: episodes_partitioned PARTITION(doctor_pt=11).title SIMPLE [(episodes)episodes.FieldSchema(name:title, type:string, comment:episode title), ] +POSTHOOK: Lineage: episodes_partitioned PARTITION(doctor_pt=1).air_date SIMPLE [(episodes)episodes.FieldSchema(name:air_date, type:string, comment:initial date), ] +POSTHOOK: Lineage: episodes_partitioned PARTITION(doctor_pt=1).doctor SIMPLE [(episodes)episodes.FieldSchema(name:doctor, type:int, comment:main actor playing the Doctor in episode), ] +POSTHOOK: Lineage: episodes_partitioned PARTITION(doctor_pt=1).title SIMPLE [(episodes)episodes.FieldSchema(name:title, type:string, comment:episode title), ] +POSTHOOK: Lineage: episodes_partitioned PARTITION(doctor_pt=2).air_date SIMPLE [(episodes)episodes.FieldSchema(name:air_date, type:string, comment:initial date), ] +POSTHOOK: Lineage: episodes_partitioned PARTITION(doctor_pt=2).doctor SIMPLE [(episodes)episodes.FieldSchema(name:doctor, type:int, comment:main actor playing the Doctor in episode), ] +POSTHOOK: Lineage: episodes_partitioned PARTITION(doctor_pt=2).title SIMPLE [(episodes)episodes.FieldSchema(name:title, type:string, comment:episode title), ] +POSTHOOK: Lineage: episodes_partitioned PARTITION(doctor_pt=4).air_date SIMPLE [(episodes)episodes.FieldSchema(name:air_date, type:string, comment:initial date), ] +POSTHOOK: Lineage: episodes_partitioned PARTITION(doctor_pt=4).doctor SIMPLE [(episodes)episodes.FieldSchema(name:doctor, type:int, comment:main actor playing the Doctor in episode), ] +POSTHOOK: Lineage: episodes_partitioned PARTITION(doctor_pt=4).title SIMPLE [(episodes)episodes.FieldSchema(name:title, type:string, comment:episode title), ] +POSTHOOK: Lineage: episodes_partitioned PARTITION(doctor_pt=5).air_date SIMPLE [(episodes)episodes.FieldSchema(name:air_date, type:string, comment:initial date), ] +POSTHOOK: Lineage: episodes_partitioned PARTITION(doctor_pt=5).doctor SIMPLE [(episodes)episodes.FieldSchema(name:doctor, type:int, comment:main actor playing the Doctor in episode), ] +POSTHOOK: Lineage: episodes_partitioned PARTITION(doctor_pt=5).title SIMPLE [(episodes)episodes.FieldSchema(name:title, type:string, comment:episode title), ] +POSTHOOK: Lineage: episodes_partitioned PARTITION(doctor_pt=6).air_date SIMPLE [(episodes)episodes.FieldSchema(name:air_date, type:string, comment:initial date), ] +POSTHOOK: Lineage: episodes_partitioned PARTITION(doctor_pt=6).doctor SIMPLE [(episodes)episodes.FieldSchema(name:doctor, type:int, comment:main actor playing the Doctor in episode), ] +POSTHOOK: Lineage: episodes_partitioned PARTITION(doctor_pt=6).title SIMPLE [(episodes)episodes.FieldSchema(name:title, type:string, comment:episode title), ] +POSTHOOK: Lineage: episodes_partitioned PARTITION(doctor_pt=9).air_date SIMPLE [(episodes)episodes.FieldSchema(name:air_date, type:string, comment:initial date), ] +POSTHOOK: Lineage: episodes_partitioned PARTITION(doctor_pt=9).doctor SIMPLE [(episodes)episodes.FieldSchema(name:doctor, type:int, comment:main actor playing the Doctor in episode), ] +POSTHOOK: Lineage: episodes_partitioned PARTITION(doctor_pt=9).title SIMPLE [(episodes)episodes.FieldSchema(name:title, type:string, comment:episode title), ] PREHOOK: query: SELECT * FROM episodes_partitioned WHERE doctor_pt > 6 PREHOOK: type: QUERY PREHOOK: Input: default@episodes_partitioned diff --git a/ql/src/test/results/clientpositive/avro_sanity_test.q.out b/ql/src/test/results/clientpositive/avro_sanity_test.q.out index 687579e69ba9d48bc471a89a7322f403eefce14a..250e661e672c51ac4f3211d80bcb7222a6f1bfad 100644 --- a/ql/src/test/results/clientpositive/avro_sanity_test.q.out +++ b/ql/src/test/results/clientpositive/avro_sanity_test.q.out @@ -72,9 +72,9 @@ PREHOOK: Input: default@doctors POSTHOOK: query: DESCRIBE doctors POSTHOOK: type: DESCTABLE POSTHOOK: Input: default@doctors -number int from deserializer -first_name string from deserializer -last_name string from deserializer +number int Order of playing the role +first_name string first name of actor playing role +last_name string last name of actor playing role PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/doctors.avro' INTO TABLE doctors PREHOOK: type: LOAD #### A masked pattern was here #### diff --git a/ql/src/test/results/clientpositive/avro_schema_evolution_native.q.out b/ql/src/test/results/clientpositive/avro_schema_evolution_native.q.out index 84d8051422b032965dbd7f9b4e0892ead76f1b67..98e541aac52637f8ad8de17eb8b1f37d55d5f57b 100644 --- a/ql/src/test/results/clientpositive/avro_schema_evolution_native.q.out +++ b/ql/src/test/results/clientpositive/avro_schema_evolution_native.q.out @@ -60,27 +60,27 @@ POSTHOOK: Output: default@episodes_partitioned@doctor_pt=4 POSTHOOK: Output: default@episodes_partitioned@doctor_pt=5 POSTHOOK: Output: default@episodes_partitioned@doctor_pt=6 POSTHOOK: Output: default@episodes_partitioned@doctor_pt=9 -POSTHOOK: Lineage: episodes_partitioned PARTITION(doctor_pt=11).air_date SIMPLE [(episodes)episodes.FieldSchema(name:air_date, type:string, comment:from deserializer), ] -POSTHOOK: Lineage: episodes_partitioned PARTITION(doctor_pt=11).doctor SIMPLE [(episodes)episodes.FieldSchema(name:doctor, type:int, comment:from deserializer), ] -POSTHOOK: Lineage: episodes_partitioned PARTITION(doctor_pt=11).title SIMPLE [(episodes)episodes.FieldSchema(name:title, type:string, comment:from deserializer), ] -POSTHOOK: Lineage: episodes_partitioned PARTITION(doctor_pt=1).air_date SIMPLE [(episodes)episodes.FieldSchema(name:air_date, type:string, comment:from deserializer), ] -POSTHOOK: Lineage: episodes_partitioned PARTITION(doctor_pt=1).doctor SIMPLE [(episodes)episodes.FieldSchema(name:doctor, type:int, comment:from deserializer), ] -POSTHOOK: Lineage: episodes_partitioned PARTITION(doctor_pt=1).title SIMPLE [(episodes)episodes.FieldSchema(name:title, type:string, comment:from deserializer), ] -POSTHOOK: Lineage: episodes_partitioned PARTITION(doctor_pt=2).air_date SIMPLE [(episodes)episodes.FieldSchema(name:air_date, type:string, comment:from deserializer), ] -POSTHOOK: Lineage: episodes_partitioned PARTITION(doctor_pt=2).doctor SIMPLE [(episodes)episodes.FieldSchema(name:doctor, type:int, comment:from deserializer), ] -POSTHOOK: Lineage: episodes_partitioned PARTITION(doctor_pt=2).title SIMPLE [(episodes)episodes.FieldSchema(name:title, type:string, comment:from deserializer), ] -POSTHOOK: Lineage: episodes_partitioned PARTITION(doctor_pt=4).air_date SIMPLE [(episodes)episodes.FieldSchema(name:air_date, type:string, comment:from deserializer), ] -POSTHOOK: Lineage: episodes_partitioned PARTITION(doctor_pt=4).doctor SIMPLE [(episodes)episodes.FieldSchema(name:doctor, type:int, comment:from deserializer), ] -POSTHOOK: Lineage: episodes_partitioned PARTITION(doctor_pt=4).title SIMPLE [(episodes)episodes.FieldSchema(name:title, type:string, comment:from deserializer), ] -POSTHOOK: Lineage: episodes_partitioned PARTITION(doctor_pt=5).air_date SIMPLE [(episodes)episodes.FieldSchema(name:air_date, type:string, comment:from deserializer), ] -POSTHOOK: Lineage: episodes_partitioned PARTITION(doctor_pt=5).doctor SIMPLE [(episodes)episodes.FieldSchema(name:doctor, type:int, comment:from deserializer), ] -POSTHOOK: Lineage: episodes_partitioned PARTITION(doctor_pt=5).title SIMPLE [(episodes)episodes.FieldSchema(name:title, type:string, comment:from deserializer), ] -POSTHOOK: Lineage: episodes_partitioned PARTITION(doctor_pt=6).air_date SIMPLE [(episodes)episodes.FieldSchema(name:air_date, type:string, comment:from deserializer), ] -POSTHOOK: Lineage: episodes_partitioned PARTITION(doctor_pt=6).doctor SIMPLE [(episodes)episodes.FieldSchema(name:doctor, type:int, comment:from deserializer), ] -POSTHOOK: Lineage: episodes_partitioned PARTITION(doctor_pt=6).title SIMPLE [(episodes)episodes.FieldSchema(name:title, type:string, comment:from deserializer), ] -POSTHOOK: Lineage: episodes_partitioned PARTITION(doctor_pt=9).air_date SIMPLE [(episodes)episodes.FieldSchema(name:air_date, type:string, comment:from deserializer), ] -POSTHOOK: Lineage: episodes_partitioned PARTITION(doctor_pt=9).doctor SIMPLE [(episodes)episodes.FieldSchema(name:doctor, type:int, comment:from deserializer), ] -POSTHOOK: Lineage: episodes_partitioned PARTITION(doctor_pt=9).title SIMPLE [(episodes)episodes.FieldSchema(name:title, type:string, comment:from deserializer), ] +POSTHOOK: Lineage: episodes_partitioned PARTITION(doctor_pt=11).air_date SIMPLE [(episodes)episodes.FieldSchema(name:air_date, type:string, comment:initial date), ] +POSTHOOK: Lineage: episodes_partitioned PARTITION(doctor_pt=11).doctor SIMPLE [(episodes)episodes.FieldSchema(name:doctor, type:int, comment:main actor playing the Doctor in episode), ] +POSTHOOK: Lineage: episodes_partitioned PARTITION(doctor_pt=11).title SIMPLE [(episodes)episodes.FieldSchema(name:title, type:string, comment:episode title), ] +POSTHOOK: Lineage: episodes_partitioned PARTITION(doctor_pt=1).air_date SIMPLE [(episodes)episodes.FieldSchema(name:air_date, type:string, comment:initial date), ] +POSTHOOK: Lineage: episodes_partitioned PARTITION(doctor_pt=1).doctor SIMPLE [(episodes)episodes.FieldSchema(name:doctor, type:int, comment:main actor playing the Doctor in episode), ] +POSTHOOK: Lineage: episodes_partitioned PARTITION(doctor_pt=1).title SIMPLE [(episodes)episodes.FieldSchema(name:title, type:string, comment:episode title), ] +POSTHOOK: Lineage: episodes_partitioned PARTITION(doctor_pt=2).air_date SIMPLE [(episodes)episodes.FieldSchema(name:air_date, type:string, comment:initial date), ] +POSTHOOK: Lineage: episodes_partitioned PARTITION(doctor_pt=2).doctor SIMPLE [(episodes)episodes.FieldSchema(name:doctor, type:int, comment:main actor playing the Doctor in episode), ] +POSTHOOK: Lineage: episodes_partitioned PARTITION(doctor_pt=2).title SIMPLE [(episodes)episodes.FieldSchema(name:title, type:string, comment:episode title), ] +POSTHOOK: Lineage: episodes_partitioned PARTITION(doctor_pt=4).air_date SIMPLE [(episodes)episodes.FieldSchema(name:air_date, type:string, comment:initial date), ] +POSTHOOK: Lineage: episodes_partitioned PARTITION(doctor_pt=4).doctor SIMPLE [(episodes)episodes.FieldSchema(name:doctor, type:int, comment:main actor playing the Doctor in episode), ] +POSTHOOK: Lineage: episodes_partitioned PARTITION(doctor_pt=4).title SIMPLE [(episodes)episodes.FieldSchema(name:title, type:string, comment:episode title), ] +POSTHOOK: Lineage: episodes_partitioned PARTITION(doctor_pt=5).air_date SIMPLE [(episodes)episodes.FieldSchema(name:air_date, type:string, comment:initial date), ] +POSTHOOK: Lineage: episodes_partitioned PARTITION(doctor_pt=5).doctor SIMPLE [(episodes)episodes.FieldSchema(name:doctor, type:int, comment:main actor playing the Doctor in episode), ] +POSTHOOK: Lineage: episodes_partitioned PARTITION(doctor_pt=5).title SIMPLE [(episodes)episodes.FieldSchema(name:title, type:string, comment:episode title), ] +POSTHOOK: Lineage: episodes_partitioned PARTITION(doctor_pt=6).air_date SIMPLE [(episodes)episodes.FieldSchema(name:air_date, type:string, comment:initial date), ] +POSTHOOK: Lineage: episodes_partitioned PARTITION(doctor_pt=6).doctor SIMPLE [(episodes)episodes.FieldSchema(name:doctor, type:int, comment:main actor playing the Doctor in episode), ] +POSTHOOK: Lineage: episodes_partitioned PARTITION(doctor_pt=6).title SIMPLE [(episodes)episodes.FieldSchema(name:title, type:string, comment:episode title), ] +POSTHOOK: Lineage: episodes_partitioned PARTITION(doctor_pt=9).air_date SIMPLE [(episodes)episodes.FieldSchema(name:air_date, type:string, comment:initial date), ] +POSTHOOK: Lineage: episodes_partitioned PARTITION(doctor_pt=9).doctor SIMPLE [(episodes)episodes.FieldSchema(name:doctor, type:int, comment:main actor playing the Doctor in episode), ] +POSTHOOK: Lineage: episodes_partitioned PARTITION(doctor_pt=9).title SIMPLE [(episodes)episodes.FieldSchema(name:title, type:string, comment:episode title), ] PREHOOK: query: ALTER TABLE episodes_partitioned SET SERDE 'org.apache.hadoop.hive.serde2.avro.AvroSerDe' WITH diff --git a/ql/src/test/results/clientpositive/avro_schema_literal.q.out b/ql/src/test/results/clientpositive/avro_schema_literal.q.out index b13694d16096a89c62aec82eceb057b47c2c4878..8ed767641bfc787455f5e300e34d533e9b2f6448 100644 --- a/ql/src/test/results/clientpositive/avro_schema_literal.q.out +++ b/ql/src/test/results/clientpositive/avro_schema_literal.q.out @@ -70,20 +70,20 @@ PREHOOK: Input: default@avro1 POSTHOOK: query: DESCRIBE avro1 POSTHOOK: type: DESCTABLE POSTHOOK: Input: default@avro1 -string1 string from deserializer -int1 int from deserializer -tinyint1 int from deserializer -smallint1 int from deserializer -bigint1 bigint from deserializer -boolean1 boolean from deserializer -float1 float from deserializer -double1 double from deserializer -list1 array from deserializer -map1 map from deserializer -struct1 struct from deserializer -union1 uniontype from deserializer -enum1 string from deserializer -nullableint int from deserializer -bytes1 binary from deserializer -fixed1 binary from deserializer -dec1 decimal(5,2) from deserializer +string1 string +int1 int +tinyint1 int +smallint1 int +bigint1 bigint +boolean1 boolean +float1 float +double1 double +list1 array +map1 map +struct1 struct +union1 uniontype +enum1 string +nullableint int +bytes1 binary +fixed1 binary +dec1 decimal(5,2) diff --git a/ql/src/test/results/clientpositive/create_like.q.out b/ql/src/test/results/clientpositive/create_like.q.out index 5e6836753effd84fc4e932ffc80a29b780613a23..27142c66ec5fd4573491fedc13a044d0678edd54 100644 --- a/ql/src/test/results/clientpositive/create_like.q.out +++ b/ql/src/test/results/clientpositive/create_like.q.out @@ -331,9 +331,9 @@ POSTHOOK: type: DESCTABLE POSTHOOK: Input: default@doctors # col_name data_type comment -number int from deserializer -first_name string from deserializer -last_name string from deserializer +number int Order of playing the role +first_name string first name of actor playing role +last_name string last name of actor playing role # Detailed Table Information Database: default @@ -380,9 +380,9 @@ POSTHOOK: type: DESCTABLE POSTHOOK: Input: default@doctors2 # col_name data_type comment -number int from deserializer -first_name string from deserializer -last_name string from deserializer +number int Order of playing the role +first_name string first name of actor playing role +last_name string last name of actor playing role # Detailed Table Information Database: default diff --git a/ql/src/test/results/clientpositive/index_serde.q.out b/ql/src/test/results/clientpositive/index_serde.q.out index c0fc7e529a70ff8abfcc68bcd4e0e40e1cf87098..472c2fe0a226b05f72dc01b484428fc4d0d55a9b 100644 --- a/ql/src/test/results/clientpositive/index_serde.q.out +++ b/ql/src/test/results/clientpositive/index_serde.q.out @@ -72,9 +72,9 @@ PREHOOK: Input: default@doctors POSTHOOK: query: DESCRIBE doctors POSTHOOK: type: DESCTABLE POSTHOOK: Input: default@doctors -number int from deserializer -first_name string from deserializer -last_name string from deserializer +number int Order of playing the role +first_name string first name of actor playing role +last_name string last name of actor playing role PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/doctors.avro' INTO TABLE doctors PREHOOK: type: LOAD #### A masked pattern was here #### @@ -98,7 +98,7 @@ PREHOOK: Input: default@default__doctors_doctors_index__ POSTHOOK: query: DESCRIBE EXTENDED default__doctors_doctors_index__ POSTHOOK: type: DESCTABLE POSTHOOK: Input: default@default__doctors_doctors_index__ -number int from deserializer +number int Order of playing the role _bucketname string _offsets array @@ -113,7 +113,7 @@ POSTHOOK: Input: default@doctors POSTHOOK: Output: default@default__doctors_doctors_index__ POSTHOOK: Lineage: default__doctors_doctors_index__._bucketname SIMPLE [(doctors)doctors.FieldSchema(name:INPUT__FILE__NAME, type:string, comment:), ] POSTHOOK: Lineage: default__doctors_doctors_index__._offsets EXPRESSION [(doctors)doctors.FieldSchema(name:BLOCK__OFFSET__INSIDE__FILE, type:bigint, comment:), ] -POSTHOOK: Lineage: default__doctors_doctors_index__.number SIMPLE [(doctors)doctors.FieldSchema(name:number, type:int, comment:from deserializer), ] +POSTHOOK: Lineage: default__doctors_doctors_index__.number SIMPLE [(doctors)doctors.FieldSchema(name:number, type:int, comment:Order of playing the role), ] PREHOOK: query: EXPLAIN SELECT * FROM doctors WHERE number > 6 PREHOOK: type: QUERY POSTHOOK: query: EXPLAIN SELECT * FROM doctors WHERE number > 6 diff --git a/ql/src/test/results/clientpositive/spark/avro_decimal_native.q.out b/ql/src/test/results/clientpositive/spark/avro_decimal_native.q.out index c8ae0fb2404c8891c4e307d82d1c2d7f180eae51..318be3da66a754de0414c2f63c2a831b33a1eba7 100644 --- a/ql/src/test/results/clientpositive/spark/avro_decimal_native.q.out +++ b/ql/src/test/results/clientpositive/spark/avro_decimal_native.q.out @@ -65,8 +65,8 @@ PREHOOK: Input: default@avro_dec POSTHOOK: query: DESC avro_dec POSTHOOK: type: DESCTABLE POSTHOOK: Input: default@avro_dec -name string from deserializer -value decimal(5,2) from deserializer +name string +value decimal(5,2) PREHOOK: query: INSERT OVERWRITE TABLE avro_dec SELECT name, value FROM dec PREHOOK: type: QUERY PREHOOK: Input: default@dec @@ -121,8 +121,8 @@ PREHOOK: Input: default@avro_dec1 POSTHOOK: query: DESC avro_dec1 POSTHOOK: type: DESCTABLE POSTHOOK: Input: default@avro_dec1 -name string from deserializer -value decimal(4,1) from deserializer +name string +value decimal(4,1) PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/dec.avro' INTO TABLE avro_dec1 PREHOOK: type: LOAD #### A masked pattern was here #### diff --git a/ql/src/test/results/clientpositive/spark/avro_joins.q.out b/ql/src/test/results/clientpositive/spark/avro_joins.q.out index 7e47fe0f25ad9b90971c26fc1fee95ceb32c0862..b9f233bb4d7619ceb3319278ef1a6a7446431f8a 100644 --- a/ql/src/test/results/clientpositive/spark/avro_joins.q.out +++ b/ql/src/test/results/clientpositive/spark/avro_joins.q.out @@ -31,7 +31,7 @@ TBLPROPERTIES ('avro.schema.literal'='{ { "name":"extra_field", "type":"string", - "doc:":"an extra field not in the original file", + "doc":"an extra field not in the original file", "default":"fishfingers and custard" } ] @@ -72,7 +72,7 @@ TBLPROPERTIES ('avro.schema.literal'='{ { "name":"extra_field", "type":"string", - "doc:":"an extra field not in the original file", + "doc":"an extra field not in the original file", "default":"fishfingers and custard" } ] @@ -86,10 +86,10 @@ PREHOOK: Input: default@doctors4 POSTHOOK: query: DESCRIBE doctors4 POSTHOOK: type: DESCTABLE POSTHOOK: Input: default@doctors4 -number int from deserializer -first_name string from deserializer -last_name string from deserializer -extra_field string from deserializer +number int Order of playing the role +first_name string first name of actor playing role +last_name string last name of actor playing role +extra_field string an extra field not in the original file PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/doctors.avro' INTO TABLE doctors4 PREHOOK: type: LOAD #### A masked pattern was here #### @@ -166,9 +166,9 @@ PREHOOK: Input: default@episodes POSTHOOK: query: DESCRIBE episodes POSTHOOK: type: DESCTABLE POSTHOOK: Input: default@episodes -title string from deserializer -air_date string from deserializer -doctor int from deserializer +title string episode title +air_date string initial date +doctor int main actor playing the Doctor in episode PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/episodes.avro' INTO TABLE episodes PREHOOK: type: LOAD #### A masked pattern was here #### diff --git a/ql/src/test/results/clientpositive/spark/avro_joins_native.q.out b/ql/src/test/results/clientpositive/spark/avro_joins_native.q.out index ca22a7279bc285600ffa40f76707fdd7e94dbb23..09a15498c8cf969d695289b8461bf57cc2e7dd17 100644 --- a/ql/src/test/results/clientpositive/spark/avro_joins_native.q.out +++ b/ql/src/test/results/clientpositive/spark/avro_joins_native.q.out @@ -28,9 +28,9 @@ PREHOOK: Input: default@doctors4 POSTHOOK: query: DESCRIBE doctors4 POSTHOOK: type: DESCTABLE POSTHOOK: Input: default@doctors4 -number int from deserializer -first_name string from deserializer -last_name string from deserializer +number int Order of playing the role +first_name string first name of actor playing role +last_name string last name of actor playing role PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/doctors.avro' INTO TABLE doctors4 PREHOOK: type: LOAD #### A masked pattern was here #### @@ -61,9 +61,9 @@ PREHOOK: Input: default@episodes POSTHOOK: query: DESCRIBE episodes POSTHOOK: type: DESCTABLE POSTHOOK: Input: default@episodes -title string from deserializer -air_date string from deserializer -doctor int from deserializer +title string episode title +air_date string initial date +doctor int main actor playing the Doctor in episode PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/episodes.avro' INTO TABLE episodes PREHOOK: type: LOAD #### A masked pattern was here #### diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroObjectInspectorGenerator.java b/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroObjectInspectorGenerator.java index 64a65433c7b476f8241e816e6c6fa20561872981..a07cf09d11dc55262cdbac7ad45862d3cf1783a1 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroObjectInspectorGenerator.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroObjectInspectorGenerator.java @@ -43,13 +43,15 @@ public class AvroObjectInspectorGenerator { final private List columnNames; final private List columnTypes; + final private List columnComments; final private ObjectInspector oi; public AvroObjectInspectorGenerator(Schema schema) throws SerDeException { verifySchemaIsARecord(schema); - this.columnNames = generateColumnNames(schema); + this.columnNames = AvroObjectInspectorGenerator.generateColumnNames(schema); this.columnTypes = SchemaToTypeInfo.generateColumnTypes(schema); + this.columnComments = AvroObjectInspectorGenerator.generateColumnComments(schema); assert columnNames.size() == columnTypes.size(); this.oi = createObjectInspector(); } @@ -80,7 +82,7 @@ private ObjectInspector createObjectInspector() throws SerDeException { for(int i = 0; i < columnNames.size(); i++) { columnOIs.add(i, createObjectInspectorWorker(columnTypes.get(i))); } - return ObjectInspectorFactory.getStandardStructObjectInspector(columnNames, columnOIs); + return ObjectInspectorFactory.getStandardStructObjectInspector(columnNames, columnOIs, columnComments); } private ObjectInspector createObjectInspectorWorker(TypeInfo ti) throws SerDeException { @@ -145,7 +147,7 @@ private boolean supportedCategories(TypeInfo ti) { c.equals(ObjectInspector.Category.UNION); } - private List generateColumnNames(Schema schema) { + public static List generateColumnNames(Schema schema) { List fields = schema.getFields(); List fieldsList = new ArrayList(fields.size()); @@ -156,4 +158,15 @@ private boolean supportedCategories(TypeInfo ti) { return fieldsList; } + public static List generateColumnComments(Schema schema) { + List fields = schema.getFields(); + List fieldComments = new ArrayList(fields.size()); + + for (Schema.Field field : fields) { + String fieldComment = field.doc() == null ? "" : field.doc(); + fieldComments.add(fieldComment); + } + + return fieldComments; + } } diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroSerDe.java b/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroSerDe.java index 8ba0692c0a24b876f6087d90aceb744d1dcd94be..efff663afd0a8ddb94d27f246122dca762eb65d4 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroSerDe.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroSerDe.java @@ -133,7 +133,9 @@ public static Schema getSchemaFromCols(Properties properties, if (columnCommentProperty == null || columnCommentProperty.isEmpty()) { columnComments = new ArrayList(); } else { - columnComments = Arrays.asList(columnCommentProperty.split(",")); + //Comments are separated by "\0" in columnCommentProperty, see method getSchema + //in MetaStoreUtils where this string columns.comments is generated + columnComments = Arrays.asList(columnCommentProperty.split("\0")); LOG.info("columnComments is " + columnCommentProperty); } if (columnNames.size() != columnTypes.size()) {