diff --git a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/data/JsonSerDe.java b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/data/JsonSerDe.java index 9b325b6..1b47b28 100644 --- a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/data/JsonSerDe.java +++ b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/data/JsonSerDe.java @@ -349,13 +349,7 @@ private Object extractCurrentField(JsonParser p, HCatFieldSchema hcatFieldSchema HCatFieldSchema valueSchema = hcatFieldSchema.getMapValueSchema().get(0); while ((valueToken = p.nextToken()) != JsonToken.END_OBJECT) { Object k = getObjectOfCorrespondingPrimitiveType(p.getCurrentName(), hcatFieldSchema.getMapKeyTypeInfo()); - Object v; - if (valueSchema.getType() == HCatFieldSchema.Type.STRUCT) { - v = extractCurrentField(p, valueSchema, false); - } else { - v = extractCurrentField(p, valueSchema, true); - } - + Object v = extractCurrentField(p, valueSchema, false); map.put(k, v); } val = map; diff --git a/hcatalog/core/src/test/java/org/apache/hive/hcatalog/data/TestJsonSerDe.java b/hcatalog/core/src/test/java/org/apache/hive/hcatalog/data/TestJsonSerDe.java index 618f39b..5ececb5 100644 --- a/hcatalog/core/src/test/java/org/apache/hive/hcatalog/data/TestJsonSerDe.java +++ b/hcatalog/core/src/test/java/org/apache/hive/hcatalog/data/TestJsonSerDe.java @@ -307,4 +307,40 @@ public void testUpperCaseKey() throws Exception { assertTrue(HCatDataCheckUtil.recordsEqual((HCatRecord)rjsd.deserialize(text2), expected2)); } + + private static HashMap createHashMapStringInteger(Object...vals) { + assertTrue(vals.length % 2 == 0); + HashMap retval = new HashMap(); + for (int idx = 0; idx < vals.length; idx += 2) { + retval.put((String) vals[idx], (Integer) vals[idx+1]); + } + return retval; + } + + public void testMapValues() throws Exception { + Configuration conf = new Configuration(); + Properties props = new Properties(); + + props.put(serdeConstants.LIST_COLUMNS, "a,b"); + props.put(serdeConstants.LIST_COLUMN_TYPES, "array,map"); + JsonSerDe rjsd = new JsonSerDe(); + SerDeUtils.initializeSerDe(rjsd, conf, props, null); + + Text text1 = new Text("{ \"a\":[\"aaa\"],\"b\":{\"bbb\":1}} "); + Text text2 = new Text("{\"a\":[\"yyy\"],\"b\":{\"zzz\":123}}"); + Text text3 = new Text("{\"a\":[\"a\"],\"b\":{\"x\":11, \"y\": 22, \"z\": null}}"); + + HCatRecord expected1 = new DefaultHCatRecord(Arrays.asList( + Arrays.asList("aaa"), + createHashMapStringInteger("bbb", 1))); + HCatRecord expected2 = new DefaultHCatRecord(Arrays.asList( + Arrays.asList("yyy"), + createHashMapStringInteger("zzz", 123))); + HCatRecord expected3 = new DefaultHCatRecord(Arrays.asList( + Arrays.asList("a"), + createHashMapStringInteger("x", 11, "y", 22, "z", null))); + + assertTrue(HCatDataCheckUtil.recordsEqual((HCatRecord)rjsd.deserialize(text1), expected1)); + assertTrue(HCatDataCheckUtil.recordsEqual((HCatRecord)rjsd.deserialize(text2), expected2)); + } } diff --git a/ql/src/test/queries/clientpositive/json_serde1.q b/ql/src/test/queries/clientpositive/json_serde1.q new file mode 100644 index 0000000..85f5af2 --- /dev/null +++ b/ql/src/test/queries/clientpositive/json_serde1.q @@ -0,0 +1,36 @@ + +add jar ${system:maven.local.repository}/org/apache/hive/hcatalog/hive-hcatalog-core/${system:hive.version}/hive-hcatalog-core-${system:hive.version}.jar; + +drop table if exists json_serde1_1; +drop table if exists json_serde1_2; + +create table json_serde1_1 (a array,b map) + row format serde 'org.apache.hive.hcatalog.data.JsonSerDe'; + +insert into table json_serde1_1 + select array('aaa'),map('aaa',1) from src limit 2; + +select * from json_serde1_1; + +create table json_serde1_2 ( + a array, + b map, + c struct, c4:map, c5:struct> +) row format serde 'org.apache.hive.hcatalog.data.JsonSerDe'; + +insert into table json_serde1_2 + select + array(3, 2, 1), + map(1, date '2001-01-01', 2, null), + named_struct( + 'c1', 123456, + 'c2', 'hello', + 'c3', array('aa', 'bb', 'cc'), + 'c4', map('abc', 123, 'xyz', 456), + 'c5', named_struct('c5_1', 'bye', 'c5_2', 88)) + from src limit 2; + +select * from json_serde1_2; + +drop table json_serde1_1; +drop table json_serde1_2; diff --git a/ql/src/test/results/clientpositive/json_serde1.q.out b/ql/src/test/results/clientpositive/json_serde1.q.out new file mode 100644 index 0000000..6235aff --- /dev/null +++ b/ql/src/test/results/clientpositive/json_serde1.q.out @@ -0,0 +1,113 @@ +PREHOOK: query: drop table if exists json_serde1_1 +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table if exists json_serde1_1 +POSTHOOK: type: DROPTABLE +PREHOOK: query: drop table if exists json_serde1_2 +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table if exists json_serde1_2 +POSTHOOK: type: DROPTABLE +PREHOOK: query: create table json_serde1_1 (a array,b map) + row format serde 'org.apache.hive.hcatalog.data.JsonSerDe' +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@json_serde1_1 +POSTHOOK: query: create table json_serde1_1 (a array,b map) + row format serde 'org.apache.hive.hcatalog.data.JsonSerDe' +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@json_serde1_1 +PREHOOK: query: insert into table json_serde1_1 + select array('aaa'),map('aaa',1) from src limit 2 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@json_serde1_1 +POSTHOOK: query: insert into table json_serde1_1 + select array('aaa'),map('aaa',1) from src limit 2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@json_serde1_1 +POSTHOOK: Lineage: json_serde1_1.a EXPRESSION [] +POSTHOOK: Lineage: json_serde1_1.b EXPRESSION [] +PREHOOK: query: select * from json_serde1_1 +PREHOOK: type: QUERY +PREHOOK: Input: default@json_serde1_1 +#### A masked pattern was here #### +POSTHOOK: query: select * from json_serde1_1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@json_serde1_1 +#### A masked pattern was here #### +["aaa"] {"aaa":1} +["aaa"] {"aaa":1} +PREHOOK: query: create table json_serde1_2 ( + a array, + b map, + c struct, c4:map, c5:struct> +) row format serde 'org.apache.hive.hcatalog.data.JsonSerDe' +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@json_serde1_2 +POSTHOOK: query: create table json_serde1_2 ( + a array, + b map, + c struct, c4:map, c5:struct> +) row format serde 'org.apache.hive.hcatalog.data.JsonSerDe' +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@json_serde1_2 +PREHOOK: query: insert into table json_serde1_2 + select + array(3, 2, 1), + map(1, date '2001-01-01', 2, null), + named_struct( + 'c1', 123456, + 'c2', 'hello', + 'c3', array('aa', 'bb', 'cc'), + 'c4', map('abc', 123, 'xyz', 456), + 'c5', named_struct('c5_1', 'bye', 'c5_2', 88)) + from src limit 2 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@json_serde1_2 +POSTHOOK: query: insert into table json_serde1_2 + select + array(3, 2, 1), + map(1, date '2001-01-01', 2, null), + named_struct( + 'c1', 123456, + 'c2', 'hello', + 'c3', array('aa', 'bb', 'cc'), + 'c4', map('abc', 123, 'xyz', 456), + 'c5', named_struct('c5_1', 'bye', 'c5_2', 88)) + from src limit 2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@json_serde1_2 +POSTHOOK: Lineage: json_serde1_2.a EXPRESSION [] +POSTHOOK: Lineage: json_serde1_2.b EXPRESSION [] +POSTHOOK: Lineage: json_serde1_2.c EXPRESSION [] +PREHOOK: query: select * from json_serde1_2 +PREHOOK: type: QUERY +PREHOOK: Input: default@json_serde1_2 +#### A masked pattern was here #### +POSTHOOK: query: select * from json_serde1_2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@json_serde1_2 +#### A masked pattern was here #### +[3,2,1] {1:"2001-01-01",2:null} {"c1":123456,"c2":"hello","c3":["aa","bb","cc"],"c4":{"xyz":456,"abc":123},"c5":{"c5_1":"bye","c5_2":88}} +[3,2,1] {1:"2001-01-01",2:null} {"c1":123456,"c2":"hello","c3":["aa","bb","cc"],"c4":{"xyz":456,"abc":123},"c5":{"c5_1":"bye","c5_2":88}} +PREHOOK: query: drop table json_serde1_1 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@json_serde1_1 +PREHOOK: Output: default@json_serde1_1 +POSTHOOK: query: drop table json_serde1_1 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@json_serde1_1 +POSTHOOK: Output: default@json_serde1_1 +PREHOOK: query: drop table json_serde1_2 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@json_serde1_2 +PREHOOK: Output: default@json_serde1_2 +POSTHOOK: query: drop table json_serde1_2 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@json_serde1_2 +POSTHOOK: Output: default@json_serde1_2