diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyMap.java b/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyMap.java index 686fc76..e6932d9 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyMap.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyMap.java @@ -23,6 +23,8 @@ import java.util.Map; import java.util.Set; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hive.serde2.lazy.objectinspector.LazyMapObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; import org.apache.hadoop.io.Text; @@ -36,6 +38,7 @@ */ public class LazyMap extends LazyNonPrimitive { + public static final Log LOG = LogFactory.getLog(LazyMap.class); /** * Whether the data is already parsed or not. */ @@ -170,15 +173,19 @@ private void parse() { valueLength[mapSize] = elementByteEnd - (keyEnd[mapSize] + 1); LazyPrimitive lazyKey = uncheckedGetKey(mapSize); if (lazyKey == null) { - continue; - } - Object key = lazyKey.getObject(); - if(!keySet.contains(key)) { - mapSize++; - keySet.add(key); - } else { + LOG.warn("skipped empty entry or entry with empty key in the representation of column with MAP type."); + //reset keyInited[mapSize] flag, since it may be set to true in the case of previous empty entry keyInited[mapSize] = false; + } else { + Object key = lazyKey.getObject(); + if(!keySet.contains(key)) { + mapSize++; + keySet.add(key); + } else { + keyInited[mapSize] = false; + } } + // reset keyValueSeparatorPosition keyValueSeparatorPosition = -1; elementByteBegin = elementByteEnd + 1; diff --git a/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazyArrayMapStruct.java b/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazyArrayMapStruct.java index ac0b583..829e65f 100644 --- a/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazyArrayMapStruct.java +++ b/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazyArrayMapStruct.java @@ -194,6 +194,114 @@ public void testLazyMap() throws Throwable { } } + /* + * test LazyMap with bad entries, e.g., empty key or empty entries + * where '[' and ']' don't exist, only for notation purpose, + * STX with value of 2 as entry separator, ETX with 3 as key/value separator + * */ + public void testLazyMapWithBadEntries() throws Throwable { + try { + { + // Map of String to String + Text nullSequence = new Text("\\N"); + ObjectInspector oi = LazyFactory.createLazyObjectInspector( + TypeInfoUtils.getTypeInfosFromTypeString("map").get( + 0), new byte[] {'\2', '\3'}, 0, nullSequence, + false, (byte) 0); + LazyMap b = (LazyMap) LazyFactory.createLazyObject(oi); + + //read friendly string: ak[EXT]av[STX]bk[ETX]bv[STX]ck[ETX]cv[STX]dk[ETX]dv + byte[] data = new byte[] { + 'a', 'k', '\3', 'a', 'v', + '\02', 'b', 'k', '\3', 'b', 'v', + '\02', 'c', 'k', '\3', 'c', 'v', + '\02', 'd', 'k', '\3', 'd', 'v'}; + TestLazyPrimitive.initLazyObject(b, data, 0, data.length); + + assertEquals(new Text("av"), ((LazyString) b + .getMapValueElement(new Text("ak"))).getWritableObject()); + assertNull(b.getMapValueElement(new Text("-1"))); + assertEquals(new Text("bv"), ((LazyString) b + .getMapValueElement(new Text("bk"))).getWritableObject()); + assertEquals(new Text("cv"), ((LazyString) b + .getMapValueElement(new Text("ck"))).getWritableObject()); + assertNull(b.getMapValueElement(new Text("-"))); + assertEquals(new Text("dv"), ((LazyString) b + .getMapValueElement(new Text("dk"))).getWritableObject()); + assertEquals(4, b.getMapSize()); + } + + { + // Map of String to String, LazyMap allows empty-string style key, e.g., {"" : null} + // or {"", ""}, but not null style key, e.g., {null:""} + Text nullSequence = new Text("\\N"); + ObjectInspector oi = LazyFactory.createLazyObjectInspector( + TypeInfoUtils.getTypeInfosFromTypeString("map").get( + 0), new byte[] {'\2', '\3'}, 0, nullSequence, + false, (byte) 0); + LazyMap b = (LazyMap) LazyFactory.createLazyObject(oi); + + //read friendly string: [STX]ak[EXT]av[STX]bk[ETX]bv[STX]ck[ETX]cv[STX]dk[ETX]dv + byte[] data = new byte[] { + '\02', 'a', 'k', '\3', 'a', 'v', + '\02', 'b', 'k', '\3', 'b', 'v', + '\02', 'c', 'k', '\3', 'c', 'v', + '\02', 'd', 'k', '\3', 'd', 'v'}; + TestLazyPrimitive.initLazyObject(b, data, 0, data.length); + + assertNull(b.getMapValueElement(new Text(""))); //{"" : null} + assertEquals(new Text("av"), ((LazyString) b + .getMapValueElement(new Text("ak"))).getWritableObject()); + assertNull(b.getMapValueElement(new Text("-1"))); + assertEquals(new Text("bv"), ((LazyString) b + .getMapValueElement(new Text("bk"))).getWritableObject()); + assertEquals(new Text("cv"), ((LazyString) b + .getMapValueElement(new Text("ck"))).getWritableObject()); + assertNull(b.getMapValueElement(new Text("-"))); + assertEquals(new Text("dv"), ((LazyString) b + .getMapValueElement(new Text("dk"))).getWritableObject()); + assertEquals(5, b.getMapSize()); + } + + { + // Map of String to String, LazyMap allows empty-string style key, e.g., {"" : null} + // or {"", ""}, but not null style key, e.g., {null:""} + Text nullSequence = new Text("\\N"); + ObjectInspector oi = LazyFactory.createLazyObjectInspector( + TypeInfoUtils.getTypeInfosFromTypeString("map").get( + 0), new byte[] {'\2', '\3'}, 0, nullSequence, + false, (byte) 0); + LazyMap b = (LazyMap) LazyFactory.createLazyObject(oi); + + //read friendly string: [ETX][STX]ak[EXT]av[STX]bk[ETX]bv[STX]ck[ETX]cv[STX]dk[ETX]dv + byte[] data = new byte[] { + '\03', + '\02', 'a', 'k', '\3', 'a', 'v', + '\02', 'b', 'k', '\3', 'b', 'v', + '\02', 'c', 'k', '\3', 'c', 'v', + '\02', 'd', 'k', '\3', 'd', 'v'}; + TestLazyPrimitive.initLazyObject(b, data, 0, data.length); + + assertEquals(new Text(""), ((LazyString) b + .getMapValueElement(new Text(""))).getWritableObject());//{"" : ""} + assertEquals(new Text("av"), ((LazyString) b + .getMapValueElement(new Text("ak"))).getWritableObject()); + assertNull(b.getMapValueElement(new Text("-1"))); + assertEquals(new Text("bv"), ((LazyString) b + .getMapValueElement(new Text("bk"))).getWritableObject()); + assertEquals(new Text("cv"), ((LazyString) b + .getMapValueElement(new Text("ck"))).getWritableObject()); + assertNull(b.getMapValueElement(new Text("-"))); + assertEquals(new Text("dv"), ((LazyString) b + .getMapValueElement(new Text("dk"))).getWritableObject()); + assertEquals(5, b.getMapSize()); + } + } catch(Throwable e) { + e.printStackTrace(); + throw e; + } + } + /** * Test the LazyMap class. */