diff --git a/hcatalog-pig-adapter/src/main/java/org/apache/hcatalog/pig/PigHCatUtil.java b/hcatalog-pig-adapter/src/main/java/org/apache/hcatalog/pig/PigHCatUtil.java index 6822fab..7e4dc9f 100644 --- a/hcatalog-pig-adapter/src/main/java/org/apache/hcatalog/pig/PigHCatUtil.java +++ b/hcatalog-pig-adapter/src/main/java/org/apache/hcatalog/pig/PigHCatUtil.java @@ -183,6 +183,12 @@ public class PigHCatUtil { .setDescription(hfs.getComment()) .setType(getPigType( hfs)) .setSchema(getBagSubSchema(hfs)); + } else if(hfs.getType() == Type.MAP) { + rfSchema = new ResourceFieldSchema() + .setName(hfs.getName()) + .setDescription(hfs.getComment()) + .setType(getPigType( hfs)) + .setSchema(getMapValueSchema(hfs)); } else { rfSchema = new ResourceFieldSchema() .setName(hfs.getName()) @@ -204,12 +210,12 @@ public class PigHCatUtil { String innerTupleName = HCatConstants.HCAT_PIG_INNER_TUPLE_NAME_DEFAULT; if (props != null && props.containsKey(HCatConstants.HCAT_PIG_INNER_TUPLE_NAME)) { innerTupleName = props.getProperty(HCatConstants.HCAT_PIG_INNER_TUPLE_NAME) - .replaceAll("FIELDNAME", hfs.getName()); + .replaceAll("FIELDNAME", (hfs.getName() == null ? "null" : hfs.getName())); } String innerFieldName = HCatConstants.HCAT_PIG_INNER_FIELD_NAME_DEFAULT; if (props != null && props.containsKey(HCatConstants.HCAT_PIG_INNER_FIELD_NAME)) { innerFieldName = props.getProperty(HCatConstants.HCAT_PIG_INNER_FIELD_NAME) - .replaceAll("FIELDNAME", hfs.getName()); + .replaceAll("FIELDNAME", (hfs.getName() == null ? "null" : hfs.getName())); } ResourceFieldSchema[] bagSubFieldSchemas = new ResourceFieldSchema[1]; @@ -248,6 +254,11 @@ public class PigHCatUtil { return s; } + private static ResourceSchema getMapValueSchema(HCatFieldSchema hfs) throws IOException { + HCatSchema mapValueHCatSchema = hfs.getMapValueSchema(); + return getResourceSchema(mapValueHCatSchema); + } + /** * @param hfs the field schema of the column * @return corresponding pig type diff --git a/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatLoaderComplexSchema.java b/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatLoaderComplexSchema.java index 4b20785..6a03bc3 100644 --- a/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatLoaderComplexSchema.java +++ b/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatLoaderComplexSchema.java @@ -113,10 +113,11 @@ public class TestHCatLoaderComplexSchema { "(" + "aa: chararray, " + "ab: long, " + - "ac: map[], " + + "ac: map[chararray], " + "ad: { t: (ada: long) }, " + "ae: { t: (aea:long, aeb: ( aeba: chararray, aebb: long)) }," + - "af: (afa: chararray, afb: long) " + + "af: (afa: chararray, afb: long), " + + "ag: map[(aga: long, agb: chararray)] " + ")," + "b: chararray, " + "c: long, " + @@ -131,7 +132,8 @@ public class TestHCatLoaderComplexSchema { "ac: map, " + "ad: array>, " + "ae: array>>," + - "af: struct " + + "af: struct, " + + "ag: map>" + ">, " + "b string, " + "c bigint, " + @@ -145,7 +147,8 @@ public class TestHCatLoaderComplexSchema { "ac: map, " + "ad: array, " + "ae: array>>," + - "af: struct " + + "af: struct, " + + "ag: map>" + ">, " + "b string, " + "c bigint, " + @@ -160,11 +163,12 @@ public class TestHCatLoaderComplexSchema { new HashMap() {{put("ac test1", "test 1");put("ac test2", "test 2");}}, b(t(3l), t(4l)), b(t(5l, t("aeba test", 6l))), - t("afa test", 7l) + t("afa test", 7l), + new HashMap() {{put("aga test1", t(8l, "agb test1"));put("aga test2", t(9l, "agb test2"));}} ), "b test", (long)i, - b(t(8l, t("dba test", 9l), b(t(10l))))); + b(t(10l, t("dba test", 11l), b(t(12l))))); data.add(t); } @@ -270,7 +274,7 @@ public class TestHCatLoaderComplexSchema { @Test public void testMapWithComplexData() throws Exception { - String pigSchema = "(a: long, b: map[])"; + String pigSchema = "(a: long, b: map[(aa: long, ab: chararray)])"; String tableSchema = "a bigint, b map>"; List data = new ArrayList();