diff --git accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/serde/AccumuloSerDe.java accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/serde/AccumuloSerDe.java index fcd819b..1473248 100644 --- accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/serde/AccumuloSerDe.java +++ accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/serde/AccumuloSerDe.java @@ -67,7 +67,7 @@ public void initialize(Configuration conf, Properties properties) throws SerDeEx cachedObjectInspector = LazyObjectInspectorFactory.getLazySimpleStructObjectInspector( serDeParams.getColumnNames(), columnObjectInspectors, serDeParams.getSeparators()[0], serDeParams.getNullSequence(), serDeParams.isLastColumnTakesRest(), - serDeParams.isEscaped(), serDeParams.getEscapeChar()); + serDeParams.isEscaped(), serDeParams.getEscapeChar(), serDeParams.isLeaveEscaped()); cachedRow = new LazyAccumuloRow((LazySimpleStructObjectInspector) cachedObjectInspector); @@ -95,7 +95,7 @@ public void initialize(Configuration conf, Properties properties) throws SerDeEx } else { columnObjectInspectors.add(LazyFactory.createLazyObjectInspector(type, serDeParams.getSeparators(), 1, serDeParams.getNullSequence(), serDeParams.isEscaped(), - serDeParams.getEscapeChar())); + serDeParams.getEscapeChar(), serDeParams.isLeaveEscaped())); } } diff --git accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/serde/DefaultAccumuloRowIdFactory.java accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/serde/DefaultAccumuloRowIdFactory.java index b51ff9b..a49f1ca 100644 --- accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/serde/DefaultAccumuloRowIdFactory.java +++ accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/serde/DefaultAccumuloRowIdFactory.java @@ -68,7 +68,8 @@ public void addDependencyJars(Configuration conf) throws IOException { @Override public ObjectInspector createRowIdObjectInspector(TypeInfo type) throws SerDeException { return LazyFactory.createLazyObjectInspector(type, serdeParams.getSeparators(), 1, - serdeParams.getNullSequence(), serdeParams.isEscaped(), serdeParams.getEscapeChar()); + serdeParams.getNullSequence(), serdeParams.isEscaped(), serdeParams.getEscapeChar(), + serdeParams.isLeaveEscaped()); } @Override diff --git accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/TestLazyAccumuloMap.java accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/TestLazyAccumuloMap.java index 2479fb4..f62890c 100644 --- accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/TestLazyAccumuloMap.java +++ accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/TestLazyAccumuloMap.java @@ -66,7 +66,7 @@ public void testStringMapWithProjection() throws SerDeException { Text nullSequence = new Text("\\N"); ObjectInspector oi = LazyFactory.createLazyObjectInspector(TypeInfoUtils .getTypeInfosFromTypeString("map").get(0), new byte[] {(byte) 1, (byte) 2}, - 0, nullSequence, false, (byte) 0); + 0, nullSequence, false, (byte) 0, false); LazyAccumuloMap map = new LazyAccumuloMap((LazyMapObjectInspector) oi); map.init(row, mapping); @@ -98,7 +98,7 @@ public void testIntMap() throws SerDeException, IOException { Text nullSequence = new Text("\\N"); ObjectInspector oi = LazyFactory.createLazyObjectInspector(TypeInfoUtils .getTypeInfosFromTypeString("map").get(0), new byte[] {(byte) 1, (byte) 2}, 0, - nullSequence, false, (byte) 0); + nullSequence, false, (byte) 0, false); LazyAccumuloMap map = new LazyAccumuloMap((LazyMapObjectInspector) oi); map.init(row, mapping); @@ -134,7 +134,7 @@ public void testBinaryIntMap() throws SerDeException, IOException { Text nullSequence = new Text("\\N"); ObjectInspector oi = LazyFactory.createLazyObjectInspector(TypeInfoUtils .getTypeInfosFromTypeString("map").get(0), new byte[] {(byte) 1, (byte) 2}, 0, - nullSequence, false, (byte) 0); + nullSequence, false, (byte) 0, false); LazyAccumuloMap map = new LazyAccumuloMap((LazyMapObjectInspector) oi); map.init(row, mapping); @@ -170,7 +170,7 @@ public void testMixedSerializationMap() throws SerDeException, IOException { Text nullSequence = new Text("\\N"); ObjectInspector oi = LazyFactory.createLazyObjectInspector(TypeInfoUtils .getTypeInfosFromTypeString("map").get(0), new byte[] {(byte) 1, (byte) 2}, 0, - nullSequence, false, (byte) 0); + nullSequence, false, (byte) 0, false); LazyAccumuloMap map = new LazyAccumuloMap((LazyMapObjectInspector) oi); map.init(row, mapping); diff --git accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/TestLazyAccumuloRow.java accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/TestLazyAccumuloRow.java index 5ecb5624..8de1c0d 100644 --- accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/TestLazyAccumuloRow.java +++ accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/TestLazyAccumuloRow.java @@ -59,7 +59,7 @@ public void testExpectedDeserializationOfColumns() throws Exception { LazySimpleStructObjectInspector objectInspector = (LazySimpleStructObjectInspector) LazyFactory .createLazyStructInspector(columns, types, LazySerDeParameters.DefaultSeparators, new Text( - "\\N"), false, false, (byte) '\\'); + "\\N"), false, false, (byte) '\\', false); DefaultAccumuloRowIdFactory rowIdFactory = new DefaultAccumuloRowIdFactory(); @@ -120,7 +120,7 @@ public void testDeserializationOfBinaryEncoding() throws Exception { LazySimpleStructObjectInspector objectInspector = (LazySimpleStructObjectInspector) LazyFactory .createLazyStructInspector(columns, types, LazySerDeParameters.DefaultSeparators, new Text( - "\\N"), false, false, (byte) '\\'); + "\\N"), false, false, (byte) '\\', false); DefaultAccumuloRowIdFactory rowIdFactory = new DefaultAccumuloRowIdFactory(); @@ -203,7 +203,7 @@ public void testNullInit() throws SerDeException { LazySimpleStructObjectInspector objectInspector = (LazySimpleStructObjectInspector) LazyFactory .createLazyStructInspector(columns, types, LazySerDeParameters.DefaultSeparators, new Text( - "\\N"), false, false, (byte) '\\'); + "\\N"), false, false, (byte) '\\', false); DefaultAccumuloRowIdFactory rowIdFactory = new DefaultAccumuloRowIdFactory(); diff --git accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/mr/TestHiveAccumuloTableOutputFormat.java accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/mr/TestHiveAccumuloTableOutputFormat.java index 1dd2b8c..7a86e3b 100644 --- accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/mr/TestHiveAccumuloTableOutputFormat.java +++ accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/mr/TestHiveAccumuloTableOutputFormat.java @@ -238,7 +238,7 @@ public void testWriteToMockInstance() throws Exception { Arrays.asList(stringTypeInfo, stringTypeInfo, stringTypeInfo), serDeParams.getSeparators(), serDeParams.getNullSequence(), serDeParams.isLastColumnTakesRest(), serDeParams.isEscaped(), - serDeParams.getEscapeChar()); + serDeParams.getEscapeChar(), serDeParams.isLeaveEscaped()); LazyStruct struct = (LazyStruct) LazyFactory.createLazyObject(structOI); @@ -329,7 +329,7 @@ public void testWriteToMockInstanceWithVisibility() throws Exception { TypeInfoFactory.stringTypeInfo, TypeInfoFactory.stringTypeInfo, TypeInfoFactory.stringTypeInfo), serDeParams.getSeparators(), serDeParams .getNullSequence(), serDeParams.isLastColumnTakesRest(), serDeParams.isEscaped(), - serDeParams.getEscapeChar()); + serDeParams.getEscapeChar(), serDeParams.isLeaveEscaped()); LazyStruct struct = (LazyStruct) LazyFactory.createLazyObject(structOI); @@ -416,17 +416,18 @@ public void testWriteMap() throws Exception { TypeInfo stringTypeInfo = TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.STRING_TYPE_NAME); LazyStringObjectInspector stringOI = (LazyStringObjectInspector) LazyFactory .createLazyObjectInspector(stringTypeInfo, new byte[] {0}, 0, - serDeParams.getNullSequence(), serDeParams.isEscaped(), serDeParams.getEscapeChar()); + serDeParams.getNullSequence(), serDeParams.isEscaped(), serDeParams.getEscapeChar(), + serDeParams.isLeaveEscaped()); LazyMapObjectInspector mapOI = LazyObjectInspectorFactory.getLazySimpleMapObjectInspector( stringOI, stringOI, (byte) ',', (byte) ':', serDeParams.getNullSequence(), - serDeParams.isEscaped(), serDeParams.getEscapeChar()); + serDeParams.isEscaped(), serDeParams.getEscapeChar(), serDeParams.isLeaveEscaped()); LazySimpleStructObjectInspector structOI = (LazySimpleStructObjectInspector) LazyObjectInspectorFactory .getLazySimpleStructObjectInspector(Arrays.asList("row", "data"), Arrays.asList(stringOI, mapOI), (byte) ' ', serDeParams.getNullSequence(), serDeParams.isLastColumnTakesRest(), serDeParams.isEscaped(), - serDeParams.getEscapeChar()); + serDeParams.getEscapeChar(), serDeParams.isLeaveEscaped()); LazyStruct struct = (LazyStruct) LazyFactory.createLazyObject(structOI); @@ -520,7 +521,7 @@ public void testBinarySerializationOnStringFallsBackToUtf8() throws Exception { Arrays.asList(stringTypeInfo, stringTypeInfo, stringTypeInfo), serDeParams.getSeparators(), serDeParams.getNullSequence(), serDeParams.isLastColumnTakesRest(), serDeParams.isEscaped(), - serDeParams.getEscapeChar()); + serDeParams.getEscapeChar(), serDeParams.isLeaveEscaped()); LazyStruct struct = (LazyStruct) LazyFactory.createLazyObject(structOI); diff --git accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/mr/TestHiveAccumuloTypes.java accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/mr/TestHiveAccumuloTypes.java index 926f572..53ee039 100644 --- accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/mr/TestHiveAccumuloTypes.java +++ accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/mr/TestHiveAccumuloTypes.java @@ -296,7 +296,7 @@ public void testBinaryTypes() throws Exception { byteRef.setData(valueBytes); LazyStringObjectInspector lazyStringOI = LazyPrimitiveObjectInspectorFactory - .getLazyStringObjectInspector(false, (byte) 0); + .getLazyStringObjectInspector(false, (byte) 0, false); LazyString lazyString = (LazyString) LazyFactory.createLazyObject(lazyStringOI); lazyString.init(byteRef, 0, valueBytes.length); @@ -653,7 +653,7 @@ public void testUtf8Types() throws Exception { byteRef.setData(valueBytes); LazyStringObjectInspector lazyStringOI = LazyPrimitiveObjectInspectorFactory - .getLazyStringObjectInspector(false, (byte) 0); + .getLazyStringObjectInspector(false, (byte) 0, false); LazyString lazyString = (LazyString) LazyFactory.createLazyObject(lazyStringOI); lazyString.init(byteRef, 0, valueBytes.length); diff --git accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/serde/DelimitedAccumuloRowIdFactory.java accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/serde/DelimitedAccumuloRowIdFactory.java index f885eba..0e0fe0d 100644 --- accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/serde/DelimitedAccumuloRowIdFactory.java +++ accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/serde/DelimitedAccumuloRowIdFactory.java @@ -65,7 +65,8 @@ public void init(AccumuloSerDeParameters accumuloSerDeParams, Properties propert @Override public ObjectInspector createRowIdObjectInspector(TypeInfo type) throws SerDeException { return LazyFactory.createLazyObjectInspector(type, new byte[] {separator}, 0, - serdeParams.getNullSequence(), serdeParams.isEscaped(), serdeParams.getEscapeChar()); + serdeParams.getNullSequence(), serdeParams.isEscaped(), serdeParams.getEscapeChar(), + serdeParams.isLeaveEscaped()); } @Override diff --git accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/serde/TestAccumuloRowSerializer.java accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/serde/TestAccumuloRowSerializer.java index 85bc960..743b657 100644 --- accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/serde/TestAccumuloRowSerializer.java +++ accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/serde/TestAccumuloRowSerializer.java @@ -114,7 +114,7 @@ public void testBinarySerialization() throws IOException, SerDeException { LazySimpleStructObjectInspector oi = (LazySimpleStructObjectInspector) LazyFactory .createLazyStructInspector(columns, types, serDeParams.getSeparators(), serDeParams.getNullSequence(), serDeParams.isLastColumnTakesRest(), - serDeParams.isEscaped(), serDeParams.getEscapeChar()); + serDeParams.isEscaped(), serDeParams.getEscapeChar(), serDeParams.isLeaveEscaped()); AccumuloRowSerializer serializer = new AccumuloRowSerializer(0, serDeParams, accumuloSerDeParams.getColumnMappings(), new ColumnVisibility(), @@ -183,7 +183,7 @@ public void testVisibilityLabel() throws IOException, SerDeException { LazySimpleStructObjectInspector oi = (LazySimpleStructObjectInspector) LazyFactory .createLazyStructInspector(columns, types, serDeParams.getSeparators(), serDeParams.getNullSequence(), serDeParams.isLastColumnTakesRest(), - serDeParams.isEscaped(), serDeParams.getEscapeChar()); + serDeParams.isEscaped(), serDeParams.getEscapeChar(), serDeParams.isLeaveEscaped()); AccumuloRowSerializer serializer = new AccumuloRowSerializer(0, serDeParams, accumuloSerDeParams.getColumnMappings(), new ColumnVisibility("foo"), @@ -256,16 +256,17 @@ public void testMapSerialization() throws IOException, SerDeException { TypeInfo stringTypeInfo = TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.STRING_TYPE_NAME); LazyStringObjectInspector stringOI = (LazyStringObjectInspector) LazyFactory .createLazyObjectInspector(stringTypeInfo, new byte[] {0}, 0, - serDeParams.getNullSequence(), serDeParams.isEscaped(), serDeParams.getEscapeChar()); + serDeParams.getNullSequence(), serDeParams.isEscaped(), serDeParams.getEscapeChar(), + serDeParams.isLeaveEscaped()); LazyMapObjectInspector mapOI = LazyObjectInspectorFactory.getLazySimpleMapObjectInspector( stringOI, stringOI, (byte) ',', (byte) ':', serDeParams.getNullSequence(), - serDeParams.isEscaped(), serDeParams.getEscapeChar()); + serDeParams.isEscaped(), serDeParams.getEscapeChar(), serDeParams.isLeaveEscaped()); LazySimpleStructObjectInspector structOI = (LazySimpleStructObjectInspector) LazyObjectInspectorFactory .getLazySimpleStructObjectInspector(columns, Arrays.asList(stringOI, mapOI), (byte) ' ', serDeParams.getNullSequence(), serDeParams.isLastColumnTakesRest(), - serDeParams.isEscaped(), serDeParams.getEscapeChar()); + serDeParams.isEscaped(), serDeParams.getEscapeChar(), serDeParams.isLeaveEscaped()); AccumuloRowSerializer serializer = new AccumuloRowSerializer(0, serDeParams, accumuloSerDeParams.getColumnMappings(), new ColumnVisibility(), diff --git accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/serde/TestAccumuloSerDe.java accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/serde/TestAccumuloSerDe.java index 58cac88..e2df5a8 100644 --- accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/serde/TestAccumuloSerDe.java +++ accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/serde/TestAccumuloSerDe.java @@ -395,22 +395,23 @@ public void testStructOfMapSerialization() throws IOException, SerDeException { TypeInfo stringTypeInfo = TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.STRING_TYPE_NAME); LazyStringObjectInspector stringOI = (LazyStringObjectInspector) LazyFactory .createLazyObjectInspector(stringTypeInfo, new byte[] {0}, 0, - serDeParams.getNullSequence(), serDeParams.isEscaped(), serDeParams.getEscapeChar()); + serDeParams.getNullSequence(), serDeParams.isEscaped(), serDeParams.getEscapeChar(), + serDeParams.isLeaveEscaped()); LazyMapObjectInspector mapOI = LazyObjectInspectorFactory.getLazySimpleMapObjectInspector( stringOI, stringOI, seps[3], seps[4], serDeParams.getNullSequence(), - serDeParams.isEscaped(), serDeParams.getEscapeChar()); + serDeParams.isEscaped(), serDeParams.getEscapeChar(), serDeParams.isLeaveEscaped()); LazySimpleStructObjectInspector rowStructOI = (LazySimpleStructObjectInspector) LazyObjectInspectorFactory .getLazySimpleStructObjectInspector(structColNames, Arrays. asList(mapOI, mapOI), (byte) seps[2], serDeParams.getNullSequence(), serDeParams.isLastColumnTakesRest(), - serDeParams.isEscaped(), serDeParams.getEscapeChar()); + serDeParams.isEscaped(), serDeParams.getEscapeChar(), serDeParams.isLeaveEscaped()); LazySimpleStructObjectInspector structOI = (LazySimpleStructObjectInspector) LazyObjectInspectorFactory .getLazySimpleStructObjectInspector(columns, Arrays.asList(rowStructOI, stringOI), seps[1], serDeParams.getNullSequence(), serDeParams.isLastColumnTakesRest(), - serDeParams.isEscaped(), serDeParams.getEscapeChar()); + serDeParams.isEscaped(), serDeParams.getEscapeChar(), serDeParams.isLeaveEscaped()); AccumuloRowSerializer serializer = new AccumuloRowSerializer(0, serDeParams, accumuloSerDeParams.getColumnMappings(), new ColumnVisibility(), diff --git accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/serde/TestDefaultAccumuloRowIdFactory.java accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/serde/TestDefaultAccumuloRowIdFactory.java index e0a8db9..f31a661 100644 --- accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/serde/TestDefaultAccumuloRowIdFactory.java +++ accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/serde/TestDefaultAccumuloRowIdFactory.java @@ -117,7 +117,8 @@ public void testBinaryStringRowId() throws SerDeException { DefaultAccumuloRowIdFactory rowIdFactory = new DefaultAccumuloRowIdFactory(); rowIdFactory.init(accumuloSerDe.getParams(), properties); - LazyStringObjectInspector oi = LazyPrimitiveObjectInspectorFactory.getLazyStringObjectInspector(false, (byte) '\\'); + LazyStringObjectInspector oi = + LazyPrimitiveObjectInspectorFactory.getLazyStringObjectInspector(false, (byte) '\\', false); LazyObjectBase lazyObj = rowIdFactory.createRowId(oi); Assert.assertNotNull(lazyObj); Assert.assertTrue(LazyString.class.isAssignableFrom(lazyObj.getClass())); diff --git contrib/src/java/org/apache/hadoop/hive/contrib/serde2/MultiDelimitSerDe.java contrib/src/java/org/apache/hadoop/hive/contrib/serde2/MultiDelimitSerDe.java index 296c449..88b9ee7 100644 --- contrib/src/java/org/apache/hadoop/hive/contrib/serde2/MultiDelimitSerDe.java +++ contrib/src/java/org/apache/hadoop/hive/contrib/serde2/MultiDelimitSerDe.java @@ -116,7 +116,7 @@ public void initialize(Configuration conf, Properties tbl) throws SerDeException .getColumnNames(), serdeParams.getColumnTypes(), serdeParams .getSeparators(), serdeParams.getNullSequence(), serdeParams .isLastColumnTakesRest(), serdeParams.isEscaped(), serdeParams - .getEscapeChar()); + .getEscapeChar(), serdeParams.isLeaveEscaped()); cachedLazyStruct = (LazyStruct) LazyFactory.createLazyObject(cachedObjectInspector); diff --git hbase-handler/src/test/org/apache/hadoop/hive/hbase/TestLazyHBaseCellMap.java hbase-handler/src/test/org/apache/hadoop/hive/hbase/TestLazyHBaseCellMap.java index a15f53c..a2d9e3f 100644 --- hbase-handler/src/test/org/apache/hadoop/hive/hbase/TestLazyHBaseCellMap.java +++ hbase-handler/src/test/org/apache/hadoop/hive/hbase/TestLazyHBaseCellMap.java @@ -44,7 +44,7 @@ public void testInitColumnPrefix() throws Exception { Text nullSequence = new Text("\\N"); ObjectInspector oi = LazyFactory.createLazyObjectInspector( TypeInfoUtils.getTypeInfosFromTypeString("map").get(0), - new byte[]{(byte) 1, (byte) 2}, 0, nullSequence, false, (byte) 0); + new byte[]{(byte) 1, (byte) 2}, 0, nullSequence, false, (byte) 0, false); LazyHBaseCellMap b = new LazyHBaseCellMap((LazyMapObjectInspector) oi); diff --git hbase-handler/src/test/org/apache/hadoop/hive/hbase/TestLazyHBaseObject.java hbase-handler/src/test/org/apache/hadoop/hive/hbase/TestLazyHBaseObject.java index 216d7ae..5046843 100644 --- hbase-handler/src/test/org/apache/hadoop/hive/hbase/TestLazyHBaseObject.java +++ hbase-handler/src/test/org/apache/hadoop/hive/hbase/TestLazyHBaseObject.java @@ -65,7 +65,7 @@ public void testLazyHBaseCellMap1() throws SerDeException { Text nullSequence = new Text("\\N"); ObjectInspector oi = LazyFactory.createLazyObjectInspector( TypeInfoUtils.getTypeInfosFromTypeString("map").get(0), - new byte[]{(byte)1, (byte)2}, 0, nullSequence, false, (byte)0); + new byte[]{(byte)1, (byte)2}, 0, nullSequence, false, (byte)0, false); LazyHBaseCellMap b = new LazyHBaseCellMap((LazyMapObjectInspector) oi); @@ -127,7 +127,7 @@ public void testLazyHBaseCellMap2() throws SerDeException { Text nullSequence = new Text("\\N"); ObjectInspector oi = LazyFactory.createLazyObjectInspector( TypeInfoUtils.getTypeInfosFromTypeString("map").get(0), - new byte[]{(byte)'#', (byte)'\t'}, 0, nullSequence, false, (byte)0); + new byte[]{(byte)'#', (byte)'\t'}, 0, nullSequence, false, (byte)0, false); LazyHBaseCellMap b = new LazyHBaseCellMap((LazyMapObjectInspector) oi); @@ -190,7 +190,8 @@ public void testLazyHBaseCellMap3() throws SerDeException { Text nullSequence = new Text("\\N"); TypeInfo mapBinaryIntKeyValue = TypeInfoUtils.getTypeInfoFromTypeString("map"); ObjectInspector oi = LazyFactory.createLazyObjectInspector( - mapBinaryIntKeyValue, new byte [] {(byte)1, (byte) 2}, 0, nullSequence, false, (byte) 0); + mapBinaryIntKeyValue, new byte [] {(byte)1, (byte) 2}, 0, nullSequence, false, (byte) 0, + false); LazyHBaseCellMap hbaseCellMap = new LazyHBaseCellMap((LazyMapObjectInspector) oi); List kvs = new ArrayList(); @@ -233,7 +234,8 @@ public void testLazyHBaseCellMap3() throws SerDeException { TypeInfo mapBinaryByteKeyValue = TypeInfoUtils.getTypeInfoFromTypeString("map"); oi = LazyFactory.createLazyObjectInspector( - mapBinaryByteKeyValue, new byte [] {(byte) 1, (byte) 2}, 0, nullSequence, false, (byte) 0); + mapBinaryByteKeyValue, new byte [] {(byte) 1, (byte) 2}, 0, nullSequence, false, (byte) 0, + false); hbaseCellMap = new LazyHBaseCellMap((LazyMapObjectInspector) oi); byte [] cfByte = "cf-byte".getBytes(); kvs.clear(); @@ -271,7 +273,8 @@ public void testLazyHBaseCellMap3() throws SerDeException { TypeInfo mapBinaryShortKeyValue = TypeInfoUtils.getTypeInfoFromTypeString("map"); oi = LazyFactory.createLazyObjectInspector( - mapBinaryShortKeyValue, new byte [] {(byte) 1, (byte) 2}, 0, nullSequence, false, (byte) 0); + mapBinaryShortKeyValue, new byte [] {(byte) 1, (byte) 2}, 0, nullSequence, false, (byte) 0, + false); hbaseCellMap = new LazyHBaseCellMap((LazyMapObjectInspector) oi); byte [] cfShort = "cf-short".getBytes(); kvs.clear(); @@ -309,7 +312,8 @@ public void testLazyHBaseCellMap3() throws SerDeException { TypeInfo mapBinaryLongKeyValue = TypeInfoUtils.getTypeInfoFromTypeString("map"); oi = LazyFactory.createLazyObjectInspector( - mapBinaryLongKeyValue, new byte [] {(byte) 1, (byte) 2}, 0, nullSequence, false, (byte) 0); + mapBinaryLongKeyValue, new byte [] {(byte) 1, (byte) 2}, 0, nullSequence, false, (byte) 0, + false); hbaseCellMap = new LazyHBaseCellMap((LazyMapObjectInspector) oi); byte [] cfLong = "cf-long".getBytes(); kvs.clear(); @@ -348,7 +352,7 @@ public void testLazyHBaseCellMap3() throws SerDeException { TypeInfoUtils.getTypeInfoFromTypeString("map"); oi = LazyFactory.createLazyObjectInspector( mapBinaryFloatKeyValue, new byte [] {(byte) 1, (byte) 2}, 0, nullSequence, false, - (byte) 0); + (byte) 0, false); hbaseCellMap = new LazyHBaseCellMap((LazyMapObjectInspector) oi); byte [] cfFloat = "cf-float".getBytes(); kvs.clear(); @@ -388,7 +392,7 @@ public void testLazyHBaseCellMap3() throws SerDeException { TypeInfoUtils.getTypeInfoFromTypeString("map"); oi = LazyFactory.createLazyObjectInspector( mapBinaryDoubleKeyValue, new byte [] {(byte) 1, (byte) 2}, 0, nullSequence, false, - (byte) 0); + (byte) 0, false); hbaseCellMap = new LazyHBaseCellMap((LazyMapObjectInspector) oi); byte [] cfDouble = "cf-double".getBytes(); kvs.clear(); @@ -427,7 +431,7 @@ public void testLazyHBaseCellMap3() throws SerDeException { TypeInfoUtils.getTypeInfoFromTypeString("map"); oi = LazyFactory.createLazyObjectInspector( mapBinaryBooleanKeyValue, new byte [] {(byte) 1, (byte) 2}, 0, nullSequence, false, - (byte) 0); + (byte) 0, false); hbaseCellMap = new LazyHBaseCellMap((LazyMapObjectInspector) oi); byte [] cfBoolean = "cf-boolean".getBytes(); kvs.clear(); @@ -483,7 +487,7 @@ public void testLazyHBaseRow1() throws SerDeException { ObjectInspector oi = LazyFactory.createLazyStructInspector(fieldNames, fieldTypeInfos, new byte[] {' ', ':', '='}, - nullSequence, false, false, (byte)0); + nullSequence, false, false, (byte)0, false); LazyHBaseRow o = new LazyHBaseRow((LazySimpleStructObjectInspector) oi, columnMappings); List kvs = new ArrayList(); @@ -609,7 +613,7 @@ public void testLazyHBaseRow2() throws SerDeException { fieldNames, fieldTypeInfos, new byte[] {' ', ':', '='}, - nullSequence, false, false, (byte) 0); + nullSequence, false, false, (byte) 0, false); LazyHBaseRow o = new LazyHBaseRow((LazySimpleStructObjectInspector) oi, columnMappings); List kvs = new ArrayList(); @@ -732,7 +736,7 @@ public void testLazyHBaseRow3() throws SerDeException { ObjectInspector oi = LazyFactory.createLazyStructInspector(fieldNames, fieldTypeInfos, - new byte [] {' ', ':', '='}, nullSequence, false, false, (byte) 0); + new byte [] {' ', ':', '='}, nullSequence, false, false, (byte) 0, false); LazyHBaseRow o = new LazyHBaseRow((LazySimpleStructObjectInspector) oi, columnMappings); diff --git serde/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/serde/serdeConstants.java serde/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/serde/serdeConstants.java index 62877f9..4763e50 100644 --- serde/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/serde/serdeConstants.java +++ serde/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/serde/serdeConstants.java @@ -58,6 +58,8 @@ public static final String SERIALIZATION_ENCODING = "serialization.encoding"; + public static final String SERIALIZATION_LEAVE_ESCAPED = "serialization.leave.escaped"; + public static final String FIELD_DELIM = "field.delim"; public static final String COLLECTION_DELIM = "collection.delim"; diff --git serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroLazyObjectInspector.java serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroLazyObjectInspector.java index ff8ac36..22abd5e 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroLazyObjectInspector.java +++ serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroLazyObjectInspector.java @@ -84,14 +84,15 @@ * row if there are extra fields. * @param escaped whether the data is escaped or not * @param escapeChar if escaped is true, the escape character + * @param leaveEscaped if data is to be output as escaped * */ @Deprecated public AvroLazyObjectInspector(List structFieldNames, List structFieldObjectInspectors, List structFieldComments, byte separator, Text nullSequence, boolean lastColumnTakesRest, boolean escaped, - byte escapeChar) { + byte escapeChar, boolean leaveEscaped) { super(structFieldNames, structFieldObjectInspectors, structFieldComments, separator, - nullSequence, lastColumnTakesRest, escaped, escapeChar); + nullSequence, lastColumnTakesRest, escaped, escapeChar, leaveEscaped); } public AvroLazyObjectInspector(List structFieldNames, diff --git serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyFactory.java serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyFactory.java index eec9de0..169057b 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyFactory.java +++ serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyFactory.java @@ -236,9 +236,9 @@ @Deprecated public static ObjectInspector createLazyObjectInspector(TypeInfo typeInfo, byte[] separators, int separatorIndex, Text nullSequence, boolean escaped, - byte escapeChar, ObjectInspectorOptions option) throws SerDeException { + byte escapeChar, boolean leaveEscaped, ObjectInspectorOptions option) throws SerDeException { return createLazyObjectInspector(typeInfo, separators, separatorIndex, nullSequence, - escaped, escapeChar, false, option); + escaped, escapeChar, leaveEscaped, false, option); } /** @@ -261,9 +261,9 @@ public static ObjectInspector createLazyObjectInspector(TypeInfo typeInfo, @Deprecated public static ObjectInspector createLazyObjectInspector(TypeInfo typeInfo, byte[] separators, int separatorIndex, Text nullSequence, boolean escaped, - byte escapeChar) throws SerDeException { + byte escapeChar, boolean leaveEscaped) throws SerDeException { return createLazyObjectInspector(typeInfo, separators, separatorIndex, nullSequence, - escaped, escapeChar, false, ObjectInspectorOptions.JAVA); + escaped, escapeChar, leaveEscaped, false, ObjectInspectorOptions.JAVA); } /** @@ -282,9 +282,9 @@ public static ObjectInspector createLazyObjectInspector(TypeInfo typeInfo, @Deprecated public static ObjectInspector createLazyObjectInspector(TypeInfo typeInfo, byte[] separators, int separatorIndex, Text nullSequence, boolean escaped, - byte escapeChar, boolean extendedBooleanLiteral) throws SerDeException { + byte escapeChar, boolean leaveEscaped, boolean extendedBooleanLiteral) throws SerDeException { return createLazyObjectInspector(typeInfo, separators, separatorIndex, nullSequence, escaped, - escapeChar, extendedBooleanLiteral, ObjectInspectorOptions.JAVA); + escapeChar, leaveEscaped, extendedBooleanLiteral, ObjectInspectorOptions.JAVA); } /** @@ -304,9 +304,10 @@ public static ObjectInspector createLazyObjectInspector(TypeInfo typeInfo, @Deprecated public static ObjectInspector createLazyObjectInspector(TypeInfo typeInfo, byte[] separators, int separatorIndex, Text nullSequence, boolean escaped, - byte escapeChar, boolean extendedBooleanLiteral, ObjectInspectorOptions option) throws SerDeException { + byte escapeChar, boolean leaveEscaped, boolean extendedBooleanLiteral, + ObjectInspectorOptions option) throws SerDeException { LazyObjectInspectorParametersImpl lazyParams = new LazyObjectInspectorParametersImpl( - escaped, escapeChar, extendedBooleanLiteral, null, separators, nullSequence); + escaped, escapeChar, leaveEscaped, extendedBooleanLiteral, null, separators, nullSequence); return createLazyObjectInspector(typeInfo, separatorIndex, lazyParams, option); } @@ -390,9 +391,9 @@ public static ObjectInspector createLazyObjectInspector(TypeInfo typeInfo, public static ObjectInspector createLazyStructInspector( List columnNames, List typeInfos, byte[] separators, Text nullSequence, boolean lastColumnTakesRest, boolean escaped, - byte escapeChar) throws SerDeException { + byte escapeChar, boolean leaveEscaped) throws SerDeException { return createLazyStructInspector(columnNames, typeInfos, separators, - nullSequence, lastColumnTakesRest, escaped, escapeChar, false); + nullSequence, lastColumnTakesRest, escaped, escapeChar, leaveEscaped, false); } /** @@ -411,9 +412,10 @@ public static ObjectInspector createLazyStructInspector( public static ObjectInspector createLazyStructInspector( List columnNames, List typeInfos, byte[] separators, Text nullSequence, boolean lastColumnTakesRest, boolean escaped, - byte escapeChar, boolean extendedBooleanLiteral) throws SerDeException { + byte escapeChar, boolean leaveEscaped, boolean extendedBooleanLiteral) throws SerDeException { LazyObjectInspectorParametersImpl lazyParams = new LazyObjectInspectorParametersImpl( - escaped, escapeChar, extendedBooleanLiteral, null, separators, nullSequence, lastColumnTakesRest); + escaped, escapeChar, leaveEscaped, extendedBooleanLiteral, null, separators, nullSequence, + lastColumnTakesRest); return createLazyStructInspector(columnNames, typeInfos, lazyParams); } @@ -452,9 +454,10 @@ public static ObjectInspector createLazyStructInspector( @Deprecated public static ObjectInspector createColumnarStructInspector( List columnNames, List columnTypes, byte[] separators, - Text nullSequence, boolean escaped, byte escapeChar) throws SerDeException { + Text nullSequence, boolean escaped, byte escapeChar, boolean leaveEscaped) + throws SerDeException { LazyObjectInspectorParametersImpl lazyParams = new LazyObjectInspectorParametersImpl( - escaped, escapeChar, false, null, separators, nullSequence); + escaped, escapeChar, leaveEscaped, false, null, separators, nullSequence); return createColumnarStructInspector(columnNames, columnTypes, lazyParams); } diff --git serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySerDeParameters.java serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySerDeParameters.java index fe3f356..4c76f06 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySerDeParameters.java +++ serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySerDeParameters.java @@ -72,6 +72,7 @@ private boolean escaped; private byte escapeChar; private boolean[] needsEscape = new boolean[256]; // A flag for each byte to indicate if escape is needed. + private boolean leaveEscaped; private boolean extendedBooleanLiteral; List timestampFormats; @@ -112,6 +113,11 @@ public LazySerDeParameters(Configuration job, Properties tbl, String serdeName) needsEscape['\r'] = true; needsEscape['\n'] = true; } + + String leaveEscapedProperty = tbl.getProperty(serdeConstants.SERIALIZATION_LEAVE_ESCAPED); + leaveEscaped = (leaveEscapedProperty != null); + } else { + leaveEscaped = false; } extendedBooleanLiteral = (job == null ? false : @@ -230,6 +236,10 @@ public void setSeparator(int index, byte separator) throws SerDeException { separators[index] = separator; } + public boolean isLeaveEscaped() { + return leaveEscaped; + } + /** * To be backward-compatible, initialize the first 3 separators to * be the given values from the table properties. The default number of separators is 8; if only diff --git serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySimpleSerDe.java serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySimpleSerDe.java index c692c39..5495607 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySimpleSerDe.java +++ serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySimpleSerDe.java @@ -70,6 +70,7 @@ serdeConstants.SERIALIZATION_LAST_COLUMN_TAKES_REST, serdeConstants.ESCAPE_CHAR, serdeConstants.SERIALIZATION_ENCODING, + serdeConstants.SERIALIZATION_LEAVE_ESCAPED, LazySerDeParameters.SERIALIZATION_EXTEND_NESTING_LEVELS, LazySerDeParameters.SERIALIZATION_EXTEND_ADDITIONAL_NESTING_LEVELS }) diff --git serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyString.java serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyString.java index 539d767..44bc648 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyString.java +++ serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyString.java @@ -37,7 +37,7 @@ public LazyString(LazyString copy) { @Override public void init(ByteArrayRef bytes, int start, int length) { - if (oi.isEscaped()) { + if (oi.isEscaped() && !oi.isLeaveEscaped()) { byte escapeChar = oi.getEscapeChar(); byte[] inputBytes = bytes.getData(); LazyUtils.copyAndEscapeStringDataToText(inputBytes, start, length, escapeChar, data); diff --git serde/src/java/org/apache/hadoop/hive/serde2/lazy/fast/LazySimpleSerializeWrite.java serde/src/java/org/apache/hadoop/hive/serde2/lazy/fast/LazySimpleSerializeWrite.java index ac567e5..5a51ba4 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/lazy/fast/LazySimpleSerializeWrite.java +++ serde/src/java/org/apache/hadoop/hive/serde2/lazy/fast/LazySimpleSerializeWrite.java @@ -67,6 +67,7 @@ private boolean[] needsEscape; private boolean isEscaped; private byte escapeChar; + private boolean isLeaveEscaped; private byte[] nullSequenceBytes; private Output output; @@ -96,6 +97,7 @@ public LazySimpleSerializeWrite(int fieldCount, separators = lazyParams.getSeparators(); isEscaped = lazyParams.isEscaped(); escapeChar = lazyParams.getEscapeChar(); + isLeaveEscaped = lazyParams.isLeaveEscaped(); needsEscape = lazyParams.getNeedsEscape(); nullSequenceBytes = lazyParams.getNullSequence().getBytes(); } @@ -232,7 +234,7 @@ public void writeDouble(double v) throws IOException { @Override public void writeString(byte[] v) throws IOException { beginPrimitive(); - LazyUtils.writeEscaped(output, v, 0, v.length, isEscaped, escapeChar, + LazyUtils.writeEscaped(output, v, 0, v.length, (isEscaped && !isLeaveEscaped), escapeChar, needsEscape); finishPrimitive(); } @@ -240,7 +242,7 @@ public void writeString(byte[] v) throws IOException { @Override public void writeString(byte[] v, int start, int length) throws IOException { beginPrimitive(); - LazyUtils.writeEscaped(output, v, start, length, isEscaped, escapeChar, + LazyUtils.writeEscaped(output, v, start, length, (isEscaped && !isLeaveEscaped), escapeChar, needsEscape); finishPrimitive(); } @@ -252,8 +254,8 @@ public void writeString(byte[] v, int start, int length) throws IOException { public void writeHiveChar(HiveChar hiveChar) throws IOException { beginPrimitive(); ByteBuffer b = Text.encode(hiveChar.getPaddedValue()); - LazyUtils.writeEscaped(output, b.array(), 0, b.limit(), isEscaped, escapeChar, - needsEscape); + LazyUtils.writeEscaped(output, b.array(), 0, b.limit(), (isEscaped && !isLeaveEscaped), + escapeChar, needsEscape); finishPrimitive(); } @@ -264,8 +266,8 @@ public void writeHiveChar(HiveChar hiveChar) throws IOException { public void writeHiveVarchar(HiveVarchar hiveVarchar) throws IOException { beginPrimitive(); ByteBuffer b = Text.encode(hiveVarchar.getValue()); - LazyUtils.writeEscaped(output, b.array(), 0, b.limit(), isEscaped, escapeChar, - needsEscape); + LazyUtils.writeEscaped(output, b.array(), 0, b.limit(), (isEscaped && !isLeaveEscaped), + escapeChar, needsEscape); finishPrimitive(); } diff --git serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazyMapObjectInspector.java serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazyMapObjectInspector.java index ff40492..47f2ca1 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazyMapObjectInspector.java +++ serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazyMapObjectInspector.java @@ -56,14 +56,14 @@ protected LazyMapObjectInspector() { protected LazyMapObjectInspector(ObjectInspector mapKeyObjectInspector, ObjectInspector mapValueObjectInspector, byte itemSeparator, byte keyValueSeparator, Text nullSequence, boolean escaped, - byte escapeChar) { + byte escapeChar, boolean leaveEscaped) { this.mapKeyObjectInspector = mapKeyObjectInspector; this.mapValueObjectInspector = mapValueObjectInspector; this.itemSeparator = itemSeparator; this.keyValueSeparator = keyValueSeparator; this.lazyParams = new LazyObjectInspectorParametersImpl( - escaped, escapeChar, false, null, null, nullSequence); + escaped, escapeChar, leaveEscaped, false, null, null, nullSequence); } protected LazyMapObjectInspector(ObjectInspector mapKeyObjectInspector, @@ -141,6 +141,10 @@ public byte getEscapeChar() { return lazyParams.getEscapeChar(); } + public boolean isLeaveEscaped() { + return lazyParams.isLeaveEscaped(); + } + public LazyObjectInspectorParameters getLazyParams() { return lazyParams; } diff --git serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazyObjectInspectorFactory.java serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazyObjectInspectorFactory.java index 4af9734..4c0be03 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazyObjectInspectorFactory.java +++ serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazyObjectInspectorFactory.java @@ -50,10 +50,10 @@ public static LazySimpleStructObjectInspector getLazySimpleStructObjectInspector List structFieldNames, List structFieldObjectInspectors, byte separator, Text nullSequence, boolean lastColumnTakesRest, boolean escaped, - byte escapeChar) { + byte escapeChar, boolean leaveEscaped) { return getLazySimpleStructObjectInspector(structFieldNames, structFieldObjectInspectors, null, separator, nullSequence, - lastColumnTakesRest, escaped, escapeChar, ObjectInspectorOptions.JAVA); + lastColumnTakesRest, escaped, escapeChar, leaveEscaped, ObjectInspectorOptions.JAVA); } @Deprecated @@ -61,10 +61,10 @@ public static LazySimpleStructObjectInspector getLazySimpleStructObjectInspector List structFieldNames, List structFieldObjectInspectors, byte separator, Text nullSequence, boolean lastColumnTakesRest, boolean escaped, - byte escapeChar, ObjectInspectorOptions option) { + byte escapeChar, boolean leaveEscaped, ObjectInspectorOptions option) { return getLazySimpleStructObjectInspector(structFieldNames, structFieldObjectInspectors, null, separator, nullSequence, - lastColumnTakesRest, escaped, escapeChar, option); + lastColumnTakesRest, escaped, escapeChar, leaveEscaped, option); } @Deprecated @@ -72,10 +72,10 @@ public static LazySimpleStructObjectInspector getLazySimpleStructObjectInspector List structFieldNames, List structFieldObjectInspectors, List structFieldComments, byte separator, Text nullSequence, boolean lastColumnTakesRest, - boolean escaped, byte escapeChar) { + boolean escaped, byte escapeChar, boolean leaveEscaped) { return getLazySimpleStructObjectInspector(structFieldNames, structFieldObjectInspectors, structFieldComments, separator, nullSequence, lastColumnTakesRest, escaped, escapeChar, - ObjectInspectorOptions.JAVA); + leaveEscaped, ObjectInspectorOptions.JAVA); } @Deprecated @@ -83,12 +83,12 @@ public static LazySimpleStructObjectInspector getLazySimpleStructObjectInspector List structFieldNames, List structFieldObjectInspectors, List structFieldComments, byte separator, Text nullSequence, boolean lastColumnTakesRest, - boolean escaped,byte escapeChar, ObjectInspectorOptions option) { + boolean escaped, byte escapeChar, boolean leaveEscaped, ObjectInspectorOptions option) { return getLazySimpleStructObjectInspector(structFieldNames, structFieldObjectInspectors, structFieldComments, separator, new LazyObjectInspectorParametersImpl( - escaped, escapeChar, false, null, null, nullSequence, lastColumnTakesRest), + escaped, escapeChar, leaveEscaped, false, null, null, nullSequence, lastColumnTakesRest), option); } @@ -140,9 +140,9 @@ public static LazySimpleStructObjectInspector getLazySimpleStructObjectInspector @Deprecated public static LazyListObjectInspector getLazySimpleListObjectInspector( ObjectInspector listElementObjectInspector, byte separator, - Text nullSequence, boolean escaped, byte escapeChar) { + Text nullSequence, boolean escaped, byte escapeChar, boolean leaveEscaped) { return getLazySimpleListObjectInspector(listElementObjectInspector, separator, - new LazyObjectInspectorParametersImpl(escaped, escapeChar, false, null, null, nullSequence)); + new LazyObjectInspectorParametersImpl(escaped, escapeChar, leaveEscaped, false, null, null, nullSequence)); } public static LazyListObjectInspector getLazySimpleListObjectInspector( @@ -175,10 +175,11 @@ public static LazyMapObjectInspector getLazySimpleMapObjectInspector( ObjectInspector mapKeyObjectInspector, ObjectInspector mapValueObjectInspector, byte itemSeparator, byte keyValueSeparator, Text nullSequence, boolean escaped, - byte escapeChar) { + byte escapeChar, boolean leaveEscaped) { return getLazySimpleMapObjectInspector(mapKeyObjectInspector, mapValueObjectInspector, itemSeparator, keyValueSeparator, - new LazyObjectInspectorParametersImpl(escaped, escapeChar, false, null, null, nullSequence)); + new LazyObjectInspectorParametersImpl(escaped, escapeChar, leaveEscaped, false, null, null, + nullSequence)); } public static LazyMapObjectInspector getLazySimpleMapObjectInspector( @@ -213,9 +214,10 @@ public static LazyMapObjectInspector getLazySimpleMapObjectInspector( @Deprecated public static LazyUnionObjectInspector getLazyUnionObjectInspector( List ois, byte separator, Text nullSequence, - boolean escaped, byte escapeChar) { + boolean escaped, byte escapeChar, boolean leaveEscaped) { return getLazyUnionObjectInspector(ois, separator, - new LazyObjectInspectorParametersImpl(escaped, escapeChar, false, null, null, nullSequence)); + new LazyObjectInspectorParametersImpl(escaped, escapeChar, leaveEscaped, false, null, null, + nullSequence)); } public static LazyUnionObjectInspector getLazyUnionObjectInspector( diff --git serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazySimpleStructObjectInspector.java serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazySimpleStructObjectInspector.java index 34f8c48..b699c43 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazySimpleStructObjectInspector.java +++ serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazySimpleStructObjectInspector.java @@ -57,18 +57,18 @@ protected LazySimpleStructObjectInspector() { protected LazySimpleStructObjectInspector(List structFieldNames, List structFieldObjectInspectors, byte separator, Text nullSequence, boolean lastColumnTakesRest, boolean escaped, - byte escapeChar) { + byte escapeChar, boolean leaveEscaped) { init(structFieldNames, structFieldObjectInspectors, null, separator, - nullSequence, lastColumnTakesRest, escaped, escapeChar); + nullSequence, lastColumnTakesRest, escaped, escapeChar, leaveEscaped); } @Deprecated public LazySimpleStructObjectInspector(List structFieldNames, List structFieldObjectInspectors, List structFieldComments, byte separator, Text nullSequence, - boolean lastColumnTakesRest, boolean escaped, byte escapeChar) { + boolean lastColumnTakesRest, boolean escaped, byte escapeChar, boolean leaveEscaped) { init(structFieldNames, structFieldObjectInspectors, structFieldComments, - separator, nullSequence, lastColumnTakesRest, escaped, escapeChar); + separator, nullSequence, lastColumnTakesRest, escaped, escapeChar, leaveEscaped); } public LazySimpleStructObjectInspector(List structFieldNames, @@ -82,9 +82,9 @@ protected void init(List structFieldNames, List structFieldObjectInspectors, List structFieldComments, byte separator, Text nullSequence, boolean lastColumnTakesRest, boolean escaped, - byte escapeChar) { + byte escapeChar, boolean leaveEscaped) { LazyObjectInspectorParameters lazyParams = - new LazyObjectInspectorParametersImpl(escaped, escapeChar, + new LazyObjectInspectorParametersImpl(escaped, escapeChar, leaveEscaped, false, null, null, nullSequence, lastColumnTakesRest); init(structFieldNames, structFieldObjectInspectors, structFieldComments, separator, lazyParams); diff --git serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyObjectInspectorParameters.java serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyObjectInspectorParameters.java index 5cda118..ac9043b 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyObjectInspectorParameters.java +++ serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyObjectInspectorParameters.java @@ -25,6 +25,7 @@ public interface LazyObjectInspectorParameters { public boolean isEscaped(); public byte getEscapeChar(); + public boolean isLeaveEscaped(); public boolean isExtendedBooleanLiteral(); public List getTimestampFormats(); public byte[] getSeparators(); diff --git serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyObjectInspectorParametersImpl.java serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyObjectInspectorParametersImpl.java index ba02f57..a7fd15a 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyObjectInspectorParametersImpl.java +++ serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyObjectInspectorParametersImpl.java @@ -30,6 +30,7 @@ LazyObjectInspectorParameters { protected boolean escaped; protected byte escapeChar; + protected boolean leaveEscaped; protected boolean extendedBooleanLiteral; protected List timestampFormats; protected byte[] separators; @@ -39,16 +40,18 @@ public LazyObjectInspectorParametersImpl() { this.escaped = false; + this.leaveEscaped = false; this.extendedBooleanLiteral = false; this.timestampFormats = null; } - public LazyObjectInspectorParametersImpl(boolean escaped, byte escapeChar, + public LazyObjectInspectorParametersImpl(boolean escaped, byte escapeChar, boolean leaveEscaped, boolean extendedBooleanLiteral, List timestampFormats, byte[] separators, Text nullSequence) { super(); this.escaped = escaped; this.escapeChar = escapeChar; + this.leaveEscaped = leaveEscaped; this.extendedBooleanLiteral = extendedBooleanLiteral; this.timestampFormats = timestampFormats; this.separators = separators; @@ -56,12 +59,13 @@ public LazyObjectInspectorParametersImpl(boolean escaped, byte escapeChar, this.lastColumnTakesRest = false; } - public LazyObjectInspectorParametersImpl(boolean escaped, byte escapeChar, + public LazyObjectInspectorParametersImpl(boolean escaped, byte escapeChar, boolean leaveEscaped, boolean extendedBooleanLiteral, List timestampFormats, byte[] separators, Text nullSequence, boolean lastColumnTakesRest) { super(); this.escaped = escaped; this.escapeChar = escapeChar; + this.leaveEscaped = leaveEscaped; this.extendedBooleanLiteral = extendedBooleanLiteral; this.timestampFormats = timestampFormats; this.separators = separators; @@ -72,6 +76,7 @@ public LazyObjectInspectorParametersImpl(boolean escaped, byte escapeChar, public LazyObjectInspectorParametersImpl(LazyObjectInspectorParameters lazyParams) { this.escaped = lazyParams.isEscaped(); this.escapeChar = lazyParams.getEscapeChar(); + this.leaveEscaped = lazyParams.isLeaveEscaped(); this.extendedBooleanLiteral = lazyParams.isExtendedBooleanLiteral(); this.timestampFormats = lazyParams.getTimestampFormats(); this.separators = lazyParams.getSeparators(); @@ -90,6 +95,11 @@ public byte getEscapeChar() { } @Override + public boolean isLeaveEscaped() { + return leaveEscaped; + } + + @Override public boolean isExtendedBooleanLiteral() { return extendedBooleanLiteral; } diff --git serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyPrimitiveObjectInspectorFactory.java serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyPrimitiveObjectInspectorFactory.java index f03bd77..109f3ea 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyPrimitiveObjectInspectorFactory.java +++ serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyPrimitiveObjectInspectorFactory.java @@ -121,14 +121,15 @@ private LazyPrimitiveObjectInspectorFactory() { } public static AbstractPrimitiveLazyObjectInspector getLazyObjectInspector( - PrimitiveTypeInfo typeInfo, boolean escaped, byte escapeChar) { - return getLazyObjectInspector(typeInfo, escaped, escapeChar, false); + PrimitiveTypeInfo typeInfo, boolean escaped, byte escapeChar, boolean leaveEscaped) { + return getLazyObjectInspector(typeInfo, escaped, escapeChar, leaveEscaped, false); } public static AbstractPrimitiveLazyObjectInspector getLazyObjectInspector( - PrimitiveTypeInfo typeInfo, boolean escaped, byte escapeChar, boolean extBoolean) { + PrimitiveTypeInfo typeInfo, boolean escaped, byte escapeChar, boolean leaveEscaped, + boolean extBoolean) { LazyObjectInspectorParameters lazyParams = new LazyObjectInspectorParametersImpl( - escaped, escapeChar, extBoolean, null, null, null); + escaped, escapeChar, leaveEscaped, extBoolean, null, null, null); return getLazyObjectInspector(typeInfo, lazyParams); } @@ -138,7 +139,8 @@ private LazyPrimitiveObjectInspectorFactory() { switch(primitiveCategory) { case STRING: - return getLazyStringObjectInspector(lazyParams.isEscaped(), lazyParams.getEscapeChar()); + return getLazyStringObjectInspector(lazyParams.isEscaped(), lazyParams.getEscapeChar(), + lazyParams.isLeaveEscaped()); case CHAR: return getLazyHiveCharObjectInspector((CharTypeInfo)typeInfo, lazyParams.isEscaped(), lazyParams.getEscapeChar()); @@ -188,15 +190,17 @@ private LazyPrimitiveObjectInspectorFactory() { return poi; } - public static LazyStringObjectInspector getLazyStringObjectInspector(boolean escaped, byte escapeChar) { + public static LazyStringObjectInspector getLazyStringObjectInspector(boolean escaped, + byte escapeChar, boolean leaveEscaped) { ArrayList signature = new ArrayList(); signature.add(TypeInfoFactory.stringTypeInfo); signature.add(Boolean.valueOf(escaped)); signature.add(Byte.valueOf(escapeChar)); + signature.add(Boolean.valueOf(leaveEscaped)); LazyStringObjectInspector result = (LazyStringObjectInspector) cachedLazyStringTypeOIs .get(signature); if (result == null) { - result = new LazyStringObjectInspector(escaped, escapeChar); + result = new LazyStringObjectInspector(escaped, escapeChar, leaveEscaped); AbstractPrimitiveLazyObjectInspector prev = cachedLazyStringTypeOIs.putIfAbsent(signature, result); if (prev != null) { diff --git serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyStringObjectInspector.java serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyStringObjectInspector.java index 28a25d6..9ae73ec 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyStringObjectInspector.java +++ serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyStringObjectInspector.java @@ -30,15 +30,17 @@ private boolean escaped; private byte escapeChar; + private boolean leaveEscaped; protected LazyStringObjectInspector() { super(); } - LazyStringObjectInspector(boolean escaped, byte escapeChar) { + LazyStringObjectInspector(boolean escaped, byte escapeChar, boolean leaveEscaped) { super(TypeInfoFactory.stringTypeInfo); this.escaped = escaped; this.escapeChar = escapeChar; + this.leaveEscaped = leaveEscaped; } @Override @@ -64,4 +66,7 @@ public byte getEscapeChar() { return escapeChar; } + public boolean isLeaveEscaped() { + return leaveEscaped; + } } diff --git serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroLazyObjectInspector.java serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroLazyObjectInspector.java index dd3e735..06b6d5c 100644 --- serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroLazyObjectInspector.java +++ serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroLazyObjectInspector.java @@ -33,44 +33,50 @@ public class TestAvroLazyObjectInspector { - @Test - public void testEmptyData(){ - List fieldNames = new ArrayList(); - fieldNames.add("myField"); + @Test + public void testEmptyData(){ + List fieldNames = new ArrayList(); + fieldNames.add("myField"); - List ois = new ArrayList(); - ois.add(LazyPrimitiveObjectInspectorFactory.getLazyStringObjectInspector(false, new Byte((byte) 0))); + List ois = new ArrayList(); + ois.add(LazyPrimitiveObjectInspectorFactory.getLazyStringObjectInspector( + false, new Byte((byte) 0), false)); - AvroLazyObjectInspector aloi = new AvroLazyObjectInspector(fieldNames, ois, null, (byte)0, new Text(), false, false, (byte)0); - LazyStruct lazyStruct = new LazyStruct(LazyObjectInspectorFactory.getLazySimpleStructObjectInspector(fieldNames, ois, (byte)0, new Text(), false, false, (byte)0)); + AvroLazyObjectInspector aloi = + new AvroLazyObjectInspector( + fieldNames, ois, null, (byte)0, new Text(), false, false, (byte)0, false); + LazyStruct lazyStruct = + new LazyStruct(LazyObjectInspectorFactory.getLazySimpleStructObjectInspector( + fieldNames, ois, (byte)0, new Text(), false, false, (byte)0, false)); - ByteArrayRef byteArrayRef = new ByteArrayRef(); - byteArrayRef.setData(new byte[0]); // set data to empty explicitly - lazyStruct.init(byteArrayRef, 0, 0); + ByteArrayRef byteArrayRef = new ByteArrayRef(); + byteArrayRef.setData(new byte[0]); // set data to empty explicitly + lazyStruct.init(byteArrayRef, 0, 0); - assertNull(aloi.getStructFieldData(lazyStruct, new TestStructField())); - } + assertNull(aloi.getStructFieldData(lazyStruct, new TestStructField())); + } - class TestStructField implements StructField { + class TestStructField implements StructField { - @Override - public String getFieldName() { - return "testfield"; - } + @Override + public String getFieldName() { + return "testfield"; + } - @Override - public ObjectInspector getFieldObjectInspector() { - return LazyPrimitiveObjectInspectorFactory.getLazyStringObjectInspector(false, new Byte((byte) 0)); - } + @Override + public ObjectInspector getFieldObjectInspector() { + return LazyPrimitiveObjectInspectorFactory. + getLazyStringObjectInspector(false, new Byte((byte) 0), false); + } - @Override - public int getFieldID() { - return 0; - } + @Override + public int getFieldID() { + return 0; + } - @Override - public String getFieldComment() { - return null; + @Override + public String getFieldComment() { + return null; } - } + } } diff --git serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazyArrayMapStruct.java serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazyArrayMapStruct.java index 5e79f7b..e629b07 100644 --- serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazyArrayMapStruct.java +++ serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazyArrayMapStruct.java @@ -61,7 +61,7 @@ public void testLazyArray() throws Throwable { Text nullSequence = new Text("\\N"); ObjectInspector oi = LazyFactory.createLazyObjectInspector(TypeInfoUtils .getTypeInfosFromTypeString("array").get(0), - new byte[] {(byte) 1}, 0, nullSequence, false, (byte) 0); + new byte[] {(byte) 1}, 0, nullSequence, false, (byte) 0, false); LazyArray b = (LazyArray) LazyFactory.createLazyObject(oi); byte[] data = new byte[] {'-', '1', 1, '\\', 'N', 1, '8'}; TestLazyPrimitive.initLazyObject(b, data, 0, data.length); @@ -83,7 +83,7 @@ public void testLazyArray() throws Throwable { // Array of String oi = LazyFactory.createLazyObjectInspector(TypeInfoUtils .getTypeInfosFromTypeString("array").get(0), - new byte[] {(byte) '\t'}, 0, nullSequence, false, (byte) 0); + new byte[] {(byte) '\t'}, 0, nullSequence, false, (byte) 0, false); b = (LazyArray) LazyFactory.createLazyObject(oi); data = new byte[] {'a', 'b', '\t', 'c', '\t', '\\', 'N', '\t', '\t', 'd'}; // Note: the first and last element of the byte[] are NOT used @@ -142,7 +142,7 @@ public void testLazyMap() throws Throwable { .createLazyObjectInspector(TypeInfoUtils .getTypeInfosFromTypeString("map").get(0), new byte[] {(byte) 1, (byte) 2}, 0, nullSequence, false, - (byte) 0); + (byte) 0, false); LazyMap b = (LazyMap) LazyFactory.createLazyObject(oi); byte[] data = new byte[] {'2', 2, 'd', 'e', 'f', 1, '-', '1', 2, '\\', 'N', 1, '0', 2, '0', 1, '8', 2, 'a', 'b', 'c'}; @@ -168,7 +168,7 @@ public void testLazyMap() throws Throwable { ObjectInspector oi = LazyFactory.createLazyObjectInspector( TypeInfoUtils.getTypeInfosFromTypeString("map").get( 0), new byte[] {(byte) '#', (byte) '\t'}, 0, nullSequence, - false, (byte) 0); + false, (byte) 0, false); LazyMap b = (LazyMap) LazyFactory.createLazyObject(oi); byte[] data = new byte[] {'2', '\t', 'd', '\t', 'f', '#', '2', '\t', 'd', '#', '-', '1', '#', '0', '\t', '0', '#', '8', '\t', 'a', 'b', 'c'}; @@ -207,7 +207,7 @@ public void testLazyMapWithBadEntries() throws Throwable { ObjectInspector oi = LazyFactory.createLazyObjectInspector( TypeInfoUtils.getTypeInfosFromTypeString("map").get( 0), new byte[] {'\2', '\3'}, 0, nullSequence, - false, (byte) 0); + false, (byte) 0, false); LazyMap b = (LazyMap) LazyFactory.createLazyObject(oi); //read friendly string: ak[EXT]av[STX]bk[ETX]bv[STX]ck[ETX]cv[STX]dk[ETX]dv @@ -238,7 +238,7 @@ public void testLazyMapWithBadEntries() throws Throwable { ObjectInspector oi = LazyFactory.createLazyObjectInspector( TypeInfoUtils.getTypeInfosFromTypeString("map").get( 0), new byte[] {'\2', '\3'}, 0, nullSequence, - false, (byte) 0); + false, (byte) 0, false); LazyMap b = (LazyMap) LazyFactory.createLazyObject(oi); //read friendly string: [STX]ak[EXT]av[STX]bk[ETX]bv[STX]ck[ETX]cv[STX]dk[ETX]dv @@ -270,7 +270,7 @@ public void testLazyMapWithBadEntries() throws Throwable { ObjectInspector oi = LazyFactory.createLazyObjectInspector( TypeInfoUtils.getTypeInfosFromTypeString("map").get( 0), new byte[] {'\2', '\3'}, 0, nullSequence, - false, (byte) 0); + false, (byte) 0, false); LazyMap b = (LazyMap) LazyFactory.createLazyObject(oi); //read friendly string: [ETX][STX]ak[EXT]av[STX]bk[ETX]bv[STX]ck[ETX]cv[STX]dk[ETX]dv @@ -313,7 +313,7 @@ public void testLazyMapWithDuplicateKeys() throws Throwable { .createLazyObjectInspector(TypeInfoUtils .getTypeInfosFromTypeString("map").get(0), new byte[] {(byte) 1, (byte) 2}, 0, nullSequence, false, - (byte) 0); + (byte) 0, false); LazyMap b = (LazyMap) LazyFactory.createLazyObject(oi); byte[] data = new byte[] {'2', 2, 'd', 'e', 'f', 1, '-', '1', 2, '\\', 'N', 1, '0', 2, '0', 1, '2', 2, 'a', 'b', 'c'}; @@ -339,7 +339,7 @@ public void testLazyMapWithDuplicateKeys() throws Throwable { ObjectInspector oi = LazyFactory.createLazyObjectInspector( TypeInfoUtils.getTypeInfosFromTypeString("map").get( 0), new byte[] {(byte) '#', (byte) '\t'}, 0, nullSequence, - false, (byte) 0); + false, (byte) 0, false); LazyMap b = (LazyMap) LazyFactory.createLazyObject(oi); byte[] data = new byte[] {'2', '\t', 'd', '\t', 'f', '#', '2', '\t', 'd', '#', '-', '1', '#', '0', '\t', '0', '#', '2', '\t', 'a', 'b', 'c'}; @@ -378,7 +378,7 @@ public void testLazyStruct() throws Throwable { ObjectInspector oi = LazyFactory.createLazyStructInspector(fieldNames, fieldTypeInfos, new byte[] {' ', ':', '='}, nullSequence, false, - false, (byte) 0); + false, (byte) 0, false); LazyStruct o = (LazyStruct) LazyFactory.createLazyObject(oi); Text data; @@ -430,7 +430,7 @@ public void testLazyStruct() throws Throwable { // test LastColumnTakesRest oi = LazyFactory.createLazyStructInspector(Arrays.asList(new String[] { "a", "b", "c", "d"}), fieldTypeInfos, - new byte[] {' ', ':', '='}, nullSequence, true, false, (byte) 0); + new byte[] {' ', ':', '='}, nullSequence, true, false, (byte) 0, false); o = (LazyStruct) LazyFactory.createLazyObject(oi); data = new Text("\\N a d=\\N:f=g:h has tail"); TestLazyPrimitive.initLazyObject(o, data.getBytes(), 0, data @@ -456,7 +456,7 @@ public void testLazyUnion() throws Throwable { Text nullSequence = new Text("\\N"); ObjectInspector oi = LazyFactory.createLazyObjectInspector(typeInfo, - new byte[] {'^', ':', '='}, 0, nullSequence, false, (byte) 0); + new byte[] {'^', ':', '='}, 0, nullSequence, false, (byte) 0, false); LazyUnion o = (LazyUnion) LazyFactory.createLazyObject(oi); Text data; diff --git serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazyPrimitive.java serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazyPrimitive.java index 663d1f2..8b2dc96 100644 --- serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazyPrimitive.java +++ serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazyPrimitive.java @@ -363,7 +363,7 @@ public void testLazyDouble() throws Throwable { public void testLazyString() throws Throwable { try { LazyString b = new LazyString(LazyPrimitiveObjectInspectorFactory - .getLazyStringObjectInspector(false, (byte) 0)); + .getLazyStringObjectInspector(false, (byte) 0, false)); initLazyObject(b, new byte[] {'0'}, 0, 0); assertEquals(new Text(""), b.getWritableObject()); initLazyObject(b, new byte[] {'0'}, 0, 1); diff --git serde/src/test/org/apache/hadoop/hive/serde2/lazybinary/TestLazyBinarySerDe.java serde/src/test/org/apache/hadoop/hive/serde2/lazybinary/TestLazyBinarySerDe.java index 70a5dd0..5ff3c96 100644 --- serde/src/test/org/apache/hadoop/hive/serde2/lazybinary/TestLazyBinarySerDe.java +++ serde/src/test/org/apache/hadoop/hive/serde2/lazybinary/TestLazyBinarySerDe.java @@ -527,7 +527,7 @@ public void testLazyBinaryObjectInspector() throws Throwable { inpBARef.setData(inpBArray); AbstractPrimitiveLazyObjectInspector binInspector = LazyPrimitiveObjectInspectorFactory - .getLazyObjectInspector(TypeInfoFactory.binaryTypeInfo, false, (byte)0); + .getLazyObjectInspector(TypeInfoFactory.binaryTypeInfo, false, (byte)0, false); //create LazyBinary initialed with inputBA LazyBinary lazyBin = (LazyBinary) LazyFactory.createLazyObject(binInspector);