diff --git a/data/files/dec_old.avro b/data/files/dec_old.avro new file mode 100644 index 0000000..bf87763 Binary files /dev/null and b/data/files/dec_old.avro differ diff --git a/ql/src/test/queries/clientnegative/avro_decimal.q b/ql/src/test/queries/clientnegative/avro_decimal.q new file mode 100644 index 0000000..538e687 --- /dev/null +++ b/ql/src/test/queries/clientnegative/avro_decimal.q @@ -0,0 +1,17 @@ +DROP TABLE IF EXISTS avro_dec; + +CREATE TABLE `avro_dec`( + `name` string COMMENT 'from deserializer', + `value` decimal(5,2) COMMENT 'from deserializer') +COMMENT 'just drop the schema right into the HQL' +ROW FORMAT SERDE + 'org.apache.hadoop.hive.serde2.avro.AvroSerDe' +STORED AS INPUTFORMAT + 'org.apache.hadoop.hive.ql.io.avro.AvroContainerInputFormat' +OUTPUTFORMAT + 'org.apache.hadoop.hive.ql.io.avro.AvroContainerOutputFormat' +TBLPROPERTIES ( + 'numFiles'='1', + 'avro.schema.literal'='{\"namespace\":\"com.howdy\",\"name\":\"some_schema\",\"type\":\"record\",\"fields\":[{\"name\":\"name\",\"type\":\"string\"},{\"name\":\"value\",\"type\":{\"type\":\"bytes\",\"logicalType\":\"decimal\",\"precision\":"5",\"scale\":"2"}}]}' +); + diff --git a/ql/src/test/queries/clientpositive/avro_decimal_old.q b/ql/src/test/queries/clientpositive/avro_decimal_old.q new file mode 100644 index 0000000..9610c47 --- /dev/null +++ b/ql/src/test/queries/clientpositive/avro_decimal_old.q @@ -0,0 +1,14 @@ +DROP TABLE IF EXISTS avro_dec_old; + +CREATE TABLE `avro_dec_old`( + `name` string COMMENT 'from deserializer', + `value` decimal(4,1) COMMENT 'from deserializer') +STORED AS AVRO; + +DESC avro_dec_old; + +LOAD DATA LOCAL INPATH '../../data/files/dec_old.avro' into TABLE avro_dec_old; + +select value from avro_dec_old; + +DROP TABLE avro_dec_old; diff --git a/ql/src/test/results/clientnegative/avro_decimal.q.out b/ql/src/test/results/clientnegative/avro_decimal.q.out new file mode 100644 index 0000000..9d00d6e --- /dev/null +++ b/ql/src/test/results/clientnegative/avro_decimal.q.out @@ -0,0 +1,22 @@ +PREHOOK: query: DROP TABLE IF EXISTS avro_dec +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE IF EXISTS avro_dec +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE `avro_dec`( + `name` string COMMENT 'from deserializer', + `value` decimal(5,2) COMMENT 'from deserializer') +COMMENT 'just drop the schema right into the HQL' +ROW FORMAT SERDE + 'org.apache.hadoop.hive.serde2.avro.AvroSerDe' +STORED AS INPUTFORMAT + 'org.apache.hadoop.hive.ql.io.avro.AvroContainerInputFormat' +OUTPUTFORMAT + 'org.apache.hadoop.hive.ql.io.avro.AvroContainerOutputFormat' +TBLPROPERTIES ( + 'numFiles'='1', + 'avro.schema.literal'='{\"namespace\":\"com.howdy\",\"name\":\"some_schema\",\"type\":\"record\",\"fields\":[{\"name\":\"name\",\"type\":\"string\"},{\"name\":\"value\",\"type\":{\"type\":\"bytes\",\"logicalType\":\"decimal\",\"precision\":"5",\"scale\":"2"}}]}' +) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@avro_dec +FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. java.lang.RuntimeException: MetaException(message:org.apache.hadoop.hive.serde2.avro.AvroSerdeException Invalid precision or scale for decimal type) diff --git a/ql/src/test/results/clientpositive/avro_decimal_old.q.out b/ql/src/test/results/clientpositive/avro_decimal_old.q.out new file mode 100644 index 0000000..22efe39 --- /dev/null +++ b/ql/src/test/results/clientpositive/avro_decimal_old.q.out @@ -0,0 +1,60 @@ +PREHOOK: query: DROP TABLE IF EXISTS avro_dec_old +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE IF EXISTS avro_dec_old +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE `avro_dec_old`( + `name` string COMMENT 'from deserializer', + `value` decimal(4,1) COMMENT 'from deserializer') +STORED AS AVRO +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@avro_dec_old +POSTHOOK: query: CREATE TABLE `avro_dec_old`( + `name` string COMMENT 'from deserializer', + `value` decimal(4,1) COMMENT 'from deserializer') +STORED AS AVRO +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@avro_dec_old +PREHOOK: query: DESC avro_dec_old +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@avro_dec_old +POSTHOOK: query: DESC avro_dec_old +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@avro_dec_old +name string from deserializer +value decimal(4,1) from deserializer +PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/dec_old.avro' into TABLE avro_dec_old +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@avro_dec_old +POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/dec_old.avro' into TABLE avro_dec_old +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@avro_dec_old +PREHOOK: query: select value from avro_dec_old +PREHOOK: type: QUERY +PREHOOK: Input: default@avro_dec_old +#### A masked pattern was here #### +POSTHOOK: query: select value from avro_dec_old +POSTHOOK: type: QUERY +POSTHOOK: Input: default@avro_dec_old +#### A masked pattern was here #### +234.8 +77.3 +55.7 +4.3 +6.0 +12.3 +33.3 +19.0 +3.2 +79.9 +PREHOOK: query: DROP TABLE avro_dec_old +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@avro_dec_old +PREHOOK: Output: default@avro_dec_old +POSTHOOK: query: DROP TABLE avro_dec_old +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@avro_dec_old +POSTHOOK: Output: default@avro_dec_old diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroDeserializer.java b/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroDeserializer.java index 4bba3d4..6165138 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroDeserializer.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroDeserializer.java @@ -244,7 +244,7 @@ private Object deserializePrimitive(Object datum, Schema fileSchema, Schema reco int scale = 0; try { - scale = fileSchema.getJsonProp(AvroSerDe.AVRO_PROP_SCALE).getIntValue(); + scale = fileSchema.getJsonProp(AvroSerDe.AVRO_PROP_SCALE).asInt(); } catch(Exception ex) { throw new AvroSerdeException("Failed to obtain scale value from file schema: " + fileSchema, ex); }