Index: serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroSerializer.java =================================================================== --- serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroSerializer.java (revision 1394076) +++ serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroSerializer.java (working copy) @@ -31,6 +31,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.Hashtable; +import java.util.HashMap; import java.util.List; import java.util.Map; @@ -206,6 +207,21 @@ assertNull(r.get("nullableint")); } + @Test + public void canSerializeMapsWithNullablePrimitiveValues() throws SerDeException, IOException { + String field = "{ \"name\":\"mapWithNulls\", \"type\": " + + "{\"type\":\"map\", \"values\": [\"null\", \"boolean\"]} }"; + + Map m = new HashMap(); + m.put("yes", true); + m.put("no", false); + m.put("maybe", null); + GenericRecord r = serializeAndDeserialize(field, "mapWithNulls", m); + + Object result = r.get("mapWithNulls"); + assertEquals(m, result); + } + @Test public void canSerializeBytes() throws SerDeException, IOException { String field = "{ \"name\":\"bytes1\", \"type\":\"bytes\" }"; Index: serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroDeserializer.java =================================================================== --- serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroDeserializer.java (revision 1394076) +++ serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroDeserializer.java (working copy) @@ -36,6 +36,7 @@ import java.util.Hashtable; import java.util.List; import java.util.Map; +import java.util.HashMap; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNull; @@ -404,6 +405,57 @@ verifyNullableType(record, s, null); } + @Test + public void canDeserializeMapWithNullablePrimitiveValues() throws SerDeException, IOException { + Schema s = Schema.parse(TestAvroObjectInspectorGenerator.MAP_WITH_NULLABLE_PRIMITIVE_VALUE_TYPE_SCHEMA); + GenericData.Record record = new GenericData.Record(s); + + Map m = new HashMap(); + m.put("one", 1l); + m.put("two", 2l); + m.put("three", 3l); + m.put("mu", null); + + record.put("aMap", m); + assertTrue(GENERIC_DATA.validate(s, record)); + System.out.println("record = " + record); + + AvroGenericRecordWritable garw = Utils.serializeAndDeserializeRecord(record); + + AvroObjectInspectorGenerator aoig = new AvroObjectInspectorGenerator(s); + + AvroDeserializer de = new AvroDeserializer(); + + ArrayList row = (ArrayList)de.deserialize(aoig.getColumnNames(), + aoig.getColumnTypes(), garw, s); + assertEquals(1, row.size()); + Object theMapObject = row.get(0); + assertTrue(theMapObject instanceof Map); + Map theMap = (Map)theMapObject; + + // Verify the raw object that's been created + assertEquals(1l, theMap.get("one")); + assertEquals(2l, theMap.get("two")); + assertEquals(3l, theMap.get("three")); + assertTrue(theMap.containsKey("mu")); + assertEquals(null, theMap.get("mu")); + + // Verify that the provided object inspector can pull out these same values + StandardStructObjectInspector oi = + (StandardStructObjectInspector)aoig.getObjectInspector(); + + List z = oi.getStructFieldsDataAsList(row); + assertEquals(1, z.size()); + StructField fieldRef = oi.getStructFieldRef("amap"); + + Map theMap2 = (Map)oi.getStructFieldData(row, fieldRef); + assertEquals(1l, theMap2.get("one")); + assertEquals(2l, theMap2.get("two")); + assertEquals(3l, theMap2.get("three")); + assertTrue(theMap2.containsKey("mu")); + assertEquals(null, theMap2.get("mu")); + } + private void verifyNullableType(GenericData.Record record, Schema s, String expected) throws SerDeException, IOException { assertTrue(GENERIC_DATA.validate(s, record)); Index: serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroObjectInspectorGenerator.java =================================================================== --- serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroObjectInspectorGenerator.java (revision 1394076) +++ serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroObjectInspectorGenerator.java (working copy) @@ -142,6 +142,18 @@ " {\"name\":\"nullableString\", \"type\":[\"null\", \"string\"]}\n" + " ]\n" + "}"; + public static final String MAP_WITH_NULLABLE_PRIMITIVE_VALUE_TYPE_SCHEMA = "{\n" + + " \"namespace\": \"testing\",\n" + + " \"name\": \"mapWithNullableUnionTest\",\n" + + " \"type\": \"record\",\n" + + " \"fields\": [\n" + + " {\n" + + " \"name\":\"aMap\",\n" + + " \"type\":{\"type\":\"map\",\n" + + " \"values\":[\"null\",\"long\"]}\n" + + "\t}\n" + + " ]\n" + + "}"; public static final String BYTES_SCHEMA = "{\n" + " \"type\": \"record\", \n" + " \"name\": \"bytesTest\",\n" + @@ -325,10 +337,19 @@ public void canHandleMapsWithPrimitiveValueTypes() throws SerDeException { Schema s = Schema.parse(MAP_WITH_PRIMITIVE_VALUE_TYPE); AvroObjectInspectorGenerator aoig = new AvroObjectInspectorGenerator(s); - + verifyMap(aoig, "aMap"); + } + + /** + * Check a given AvroObjectInspectorGenerator to verify that it matches our test + * schema's expected map. + * @param aoig should already have been intitialized, may not be null + * @param fieldName name of the contianed column, will always fail if null. + */ + private void verifyMap(final AvroObjectInspectorGenerator aoig, final String fieldName) { // Column names assertEquals(1, aoig.getColumnNames().size()); - assertEquals("aMap", aoig.getColumnNames().get(0)); + assertEquals(fieldName, aoig.getColumnNames().get(0)); // Column types assertEquals(1, aoig.getColumnTypes().size()); @@ -483,6 +504,13 @@ assertEquals(PrimitiveObjectInspector.PrimitiveCategory.STRING, pti.getPrimitiveCategory()); } + @Test // That Union[T, NULL] is converted to just T, within a Map + public void convertsMapsWithNullablePrimitiveTypes() throws SerDeException { + Schema s = Schema.parse(MAP_WITH_NULLABLE_PRIMITIVE_VALUE_TYPE_SCHEMA); + AvroObjectInspectorGenerator aoig = new AvroObjectInspectorGenerator(s); + verifyMap(aoig, "aMap"); + } + @Test public void objectInspectorsAreCached() throws SerDeException { // Verify that Hive is caching the object inspectors for us. Index: serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroSerializer.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroSerializer.java (revision 1394076) +++ serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroSerializer.java (working copy) @@ -39,7 +39,7 @@ import java.nio.ByteBuffer; import java.util.ArrayList; -import java.util.Hashtable; +import java.util.HashMap; import java.util.List; import java.util.Map; @@ -229,7 +229,7 @@ Map map = fieldOI.getMap(structFieldData); Schema valueType = schema.getValueType(); - Map deserialized = new Hashtable(fieldOI.getMapSize(structFieldData)); + Map deserialized = new HashMap(fieldOI.getMapSize(structFieldData)); for (Map.Entry entry : map.entrySet()) { deserialized.put(serialize(mapKeyTypeInfo, mapKeyObjectInspector, entry.getKey(), null), // This works, but is a bit fragile. Construct a single String schema? Index: serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroDeserializer.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroDeserializer.java (revision 1394076) +++ serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroDeserializer.java (working copy) @@ -43,7 +43,7 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.util.ArrayList; -import java.util.Hashtable; +import java.util.HashMap; import java.util.List; import java.util.Map; @@ -246,7 +246,7 @@ throws AvroSerdeException { // Avro only allows maps with Strings for keys, so we only have to worry // about deserializing the values - Map map = new Hashtable(); + Map map = new HashMap(); Map mapDatum = (Map)datum; Schema valueSchema = mapSchema.getValueType(); TypeInfo valueTypeInfo = columnType.getMapValueTypeInfo();