diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties index be4f84d..fcedcfe 100644 --- a/itests/src/test/resources/testconfiguration.properties +++ b/itests/src/test/resources/testconfiguration.properties @@ -310,6 +310,8 @@ minimr.query.negative.files=cluster_tasklog_retrieval.q,\ udf_local_resource.q spark.query.files=spark_test.q \ + avro_compression_enabled_native.q \ + avro_decimal_native.q \ timestamp_1.q \ timestamp_2.q \ timestamp_3.q \ diff --git a/ql/src/test/results/clientpositive/spark/avro_compression_enabled_native.q.out b/ql/src/test/results/clientpositive/spark/avro_compression_enabled_native.q.out new file mode 100644 index 0000000..687f17e --- /dev/null +++ b/ql/src/test/results/clientpositive/spark/avro_compression_enabled_native.q.out @@ -0,0 +1,39 @@ +PREHOOK: query: -- verify that new joins bring in correct schemas (including evolved schemas) + +CREATE TABLE doctors4 ( + number int, + first_name string, + last_name string, + extra_field string) +STORED AS AVRO +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@doctors4 +POSTHOOK: query: -- verify that new joins bring in correct schemas (including evolved schemas) + +CREATE TABLE doctors4 ( + number int, + first_name string, + last_name string, + extra_field string) +STORED AS AVRO +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@doctors4 +PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/doctors.avro' INTO TABLE doctors4 +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@doctors4 +POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/doctors.avro' INTO TABLE doctors4 +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@doctors4 +PREHOOK: query: SELECT count(*) FROM src +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: SELECT count(*) FROM src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +500 diff --git a/ql/src/test/results/clientpositive/spark/avro_decimal_native.q.out b/ql/src/test/results/clientpositive/spark/avro_decimal_native.q.out new file mode 100644 index 0000000..58a4f5c --- /dev/null +++ b/ql/src/test/results/clientpositive/spark/avro_decimal_native.q.out @@ -0,0 +1,176 @@ +PREHOOK: query: DROP TABLE IF EXISTS dec +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE IF EXISTS dec +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE dec ( + name string, + value decimal(8,4)) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@dec +POSTHOOK: query: CREATE TABLE dec ( + name string, + value decimal(8,4)) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@dec +PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/dec.txt' into TABLE dec +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@dec +POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/dec.txt' into TABLE dec +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@dec +PREHOOK: query: ANALYZE TABLE dec COMPUTE STATISTICS FOR COLUMNS value +PREHOOK: type: QUERY +PREHOOK: Input: default@dec +#### A masked pattern was here #### +POSTHOOK: query: ANALYZE TABLE dec COMPUTE STATISTICS FOR COLUMNS value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@dec +#### A masked pattern was here #### +PREHOOK: query: DESC FORMATTED dec value +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@dec +POSTHOOK: query: DESC FORMATTED dec value +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@dec +# col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment + +value decimal(8,4) -12.25 234.79 0 6 from deserializer +PREHOOK: query: DROP TABLE IF EXISTS avro_dec +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE IF EXISTS avro_dec +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE avro_dec( + name string, + value decimal(5,2)) +COMMENT 'just drop the schema right into the HQL' +STORED AS AVRO +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@avro_dec +POSTHOOK: query: CREATE TABLE avro_dec( + name string, + value decimal(5,2)) +COMMENT 'just drop the schema right into the HQL' +STORED AS AVRO +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@avro_dec +PREHOOK: query: DESC avro_dec +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@avro_dec +POSTHOOK: query: DESC avro_dec +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@avro_dec +name string from deserializer +value decimal(5,2) from deserializer +PREHOOK: query: INSERT OVERWRITE TABLE avro_dec SELECT name, value FROM dec +PREHOOK: type: QUERY +PREHOOK: Input: default@dec +PREHOOK: Output: default@avro_dec +[Error 30017]: Skipping stats aggregation by error org.apache.hadoop.hive.ql.metadata.HiveException: [Error 30015]: Stats aggregator of type counter cannot be connected to +POSTHOOK: query: INSERT OVERWRITE TABLE avro_dec SELECT name, value FROM dec +POSTHOOK: type: QUERY +POSTHOOK: Input: default@dec +POSTHOOK: Output: default@avro_dec +POSTHOOK: Lineage: avro_dec.name SIMPLE [(dec)dec.FieldSchema(name:name, type:string, comment:null), ] +POSTHOOK: Lineage: avro_dec.value EXPRESSION [(dec)dec.FieldSchema(name:value, type:decimal(8,4), comment:null), ] +PREHOOK: query: SELECT * FROM avro_dec +PREHOOK: type: QUERY +PREHOOK: Input: default@avro_dec +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM avro_dec +POSTHOOK: type: QUERY +POSTHOOK: Input: default@avro_dec +#### A masked pattern was here #### +Tom 234.79 +Beck 77.34 +Snow 55.71 +Mary 4.33 +Cluck 5.96 +Tom -12.25 +Mary 33.33 +Tom 19 +Beck 0 +Beck 79.9 +PREHOOK: query: DROP TABLE IF EXISTS avro_dec1 +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE IF EXISTS avro_dec1 +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE avro_dec1( + name string, + value decimal(4,1)) +COMMENT 'just drop the schema right into the HQL' +STORED AS AVRO +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@avro_dec1 +POSTHOOK: query: CREATE TABLE avro_dec1( + name string, + value decimal(4,1)) +COMMENT 'just drop the schema right into the HQL' +STORED AS AVRO +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@avro_dec1 +PREHOOK: query: DESC avro_dec1 +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@avro_dec1 +POSTHOOK: query: DESC avro_dec1 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@avro_dec1 +name string from deserializer +value decimal(4,1) from deserializer +PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/dec.avro' INTO TABLE avro_dec1 +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@avro_dec1 +POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/dec.avro' INTO TABLE avro_dec1 +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@avro_dec1 +PREHOOK: query: SELECT value FROM avro_dec1 +PREHOOK: type: QUERY +PREHOOK: Input: default@avro_dec1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT value FROM avro_dec1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@avro_dec1 +#### A masked pattern was here #### +234.8 +77.3 +55.7 +4.3 +6 +12.3 +33.3 +19 +3.2 +79.9 +PREHOOK: query: DROP TABLE dec +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@dec +PREHOOK: Output: default@dec +POSTHOOK: query: DROP TABLE dec +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@dec +POSTHOOK: Output: default@dec +PREHOOK: query: DROP TABLE avro_dec +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@avro_dec +PREHOOK: Output: default@avro_dec +POSTHOOK: query: DROP TABLE avro_dec +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@avro_dec +POSTHOOK: Output: default@avro_dec +PREHOOK: query: DROP TABLE avro_dec1 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@avro_dec1 +PREHOOK: Output: default@avro_dec1 +POSTHOOK: query: DROP TABLE avro_dec1 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@avro_dec1 +POSTHOOK: Output: default@avro_dec1