Index: serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyBinary.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyBinary.java (revision 1190202) +++ serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyBinary.java (working copy) @@ -18,12 +18,19 @@ package org.apache.hadoop.hive.serde2.lazy; +import java.nio.charset.CharacterCodingException; + import org.apache.commons.codec.binary.Base64; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyBinaryObjectInspector; import org.apache.hadoop.io.BytesWritable; +import org.apache.hadoop.io.Text; public class LazyBinary extends LazyPrimitive { + private static final Log LOG = LogFactory.getLog(LazyBinary.class); + LazyBinary(LazyBinaryObjectInspector oi) { super(oi); data = new BytesWritable(); @@ -42,7 +49,12 @@ byte[] recv = new byte[length]; System.arraycopy(bytes.getData(), start, recv, 0, length); - byte[] decoded = Base64.isArrayByteBase64(recv) ? Base64.decodeBase64(recv) : recv; + boolean arrayByteBase64 = Base64.isArrayByteBase64(recv); + if (arrayByteBase64) { + LOG.debug("Data not contains valid characters within the Base64 alphabet so " + + "decoded the data."); + } + byte[] decoded = arrayByteBase64 ? Base64.decodeBase64(recv) : recv; data.set(decoded, 0, decoded.length); } Index: serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyBoolean.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyBoolean.java (revision 1190202) +++ serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyBoolean.java (working copy) @@ -64,6 +64,7 @@ isNull = false; } else { isNull = true; + logExceptionMessage(bytes, start, length, "BOOLEAN"); } } Index: serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyByte.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyByte.java (revision 1190202) +++ serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyByte.java (working copy) @@ -53,6 +53,7 @@ isNull = false; } catch (NumberFormatException e) { isNull = true; + logExceptionMessage(bytes, start, length, "TINYINT"); } } Index: serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyDouble.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyDouble.java (revision 1190202) +++ serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyDouble.java (working copy) @@ -19,6 +19,8 @@ import java.nio.charset.CharacterCodingException; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hive.serde2.io.DoubleWritable; import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyDoubleObjectInspector; import org.apache.hadoop.io.Text; @@ -30,6 +32,7 @@ public class LazyDouble extends LazyPrimitive { + private static final Log LOG = LogFactory.getLog(LazyDouble.class); public LazyDouble(LazyDoubleObjectInspector oi) { super(oi); data = new DoubleWritable(); @@ -42,13 +45,18 @@ @Override public void init(ByteArrayRef bytes, int start, int length) { + String byteData = null; try { - data.set(Double.parseDouble(Text.decode(bytes.getData(), start, length))); + byteData = Text.decode(bytes.getData(), start, length); + data.set(Double.parseDouble(byteData)); isNull = false; } catch (NumberFormatException e) { isNull = true; + LOG.debug("Data not in the Double data type range so converted to null. Given data is :" + + byteData, e); } catch (CharacterCodingException e) { isNull = true; + LOG.debug("Data not in the Double data type range so converted to null.", e); } } Index: serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyFloat.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyFloat.java (revision 1190202) +++ serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyFloat.java (working copy) @@ -19,6 +19,8 @@ import java.nio.charset.CharacterCodingException; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyFloatObjectInspector; import org.apache.hadoop.io.FloatWritable; import org.apache.hadoop.io.Text; @@ -30,6 +32,7 @@ public class LazyFloat extends LazyPrimitive { + private static final Log LOG = LogFactory.getLog(LazyFloat.class); public LazyFloat(LazyFloatObjectInspector oi) { super(oi); data = new FloatWritable(); @@ -42,13 +45,18 @@ @Override public void init(ByteArrayRef bytes, int start, int length) { + String byteData = null; try { - data.set(Float.parseFloat(Text.decode(bytes.getData(), start, length))); + byteData = Text.decode(bytes.getData(), start, length); + data.set(Float.parseFloat(byteData)); isNull = false; } catch (NumberFormatException e) { isNull = true; + LOG.debug("Data not in the Float data type range so converted to null. Given data is :" + + byteData, e); } catch (CharacterCodingException e) { isNull = true; + LOG.debug("Data not in the Float data type range so converted to null.", e); } } Index: serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyInteger.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyInteger.java (revision 1190202) +++ serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyInteger.java (working copy) @@ -56,6 +56,7 @@ isNull = false; } catch (NumberFormatException e) { isNull = true; + logExceptionMessage(bytes, start, length, "INT"); } } Index: serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyLong.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyLong.java (revision 1190202) +++ serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyLong.java (working copy) @@ -56,6 +56,7 @@ isNull = false; } catch (NumberFormatException e) { isNull = true; + logExceptionMessage(bytes, start, length, "BIGINT"); } } Index: serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyPrimitive.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyPrimitive.java (revision 1190202) +++ serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyPrimitive.java (working copy) @@ -17,7 +17,12 @@ */ package org.apache.hadoop.hive.serde2.lazy; +import java.nio.charset.CharacterCodingException; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.io.Text; import org.apache.hadoop.io.Writable; /** @@ -26,6 +31,7 @@ public abstract class LazyPrimitive extends LazyObject { + private static final Log LOG = LogFactory.getLog(LazyPrimitive.class); LazyPrimitive(OI oi) { super(oi); } @@ -61,4 +67,15 @@ return isNull ? 0 : data.hashCode(); } + public void logExceptionMessage(ByteArrayRef bytes, int start, int length, String dataType) { + try { + String byteData = Text.decode(bytes.getData(), start, length); + LOG.debug("Data not in the " + dataType + + " data type range so converted to null. Given data is :" + + byteData); + } catch (CharacterCodingException e1) { + LOG.debug("Data not in the " + dataType + " data type range so converted to null.", e1); + } + } + } Index: serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyShort.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyShort.java (revision 1190202) +++ serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyShort.java (working copy) @@ -53,6 +53,7 @@ isNull = false; } catch (NumberFormatException e) { isNull = true; + logExceptionMessage(bytes, start, length, "SMALLINT"); } } Index: serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyTimestamp.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyTimestamp.java (revision 1190202) +++ serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyTimestamp.java (working copy) @@ -69,6 +69,7 @@ Timestamp t; if (s.compareTo("NULL") == 0) { t = null; + logExceptionMessage(bytes, start, length, "TIMESTAMP"); } else { t = Timestamp.valueOf(s); } Index: serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazyPrimitive.java =================================================================== --- serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazyPrimitive.java (revision 1190202) +++ serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazyPrimitive.java (working copy) @@ -375,16 +375,17 @@ } public void testLazyBinary() { - LazyBinary ba = new LazyBinary(LazyPrimitiveObjectInspectorFactory.LAZY_BINARY_OBJECT_INSPECTOR); - initLazyObject(ba, new byte[]{}, 0, 0); + initLazyObject(ba, new byte[] {}, 0, 0); assertEquals(new BytesWritable(), ba.getWritableObject()); - initLazyObject(ba, new byte[]{'%'}, 0, 1); - assertEquals(new BytesWritable(new byte[]{'%'}), ba.getWritableObject()); - initLazyObject(ba, new byte[]{'2','>','3'}, 1, 1); - assertEquals(new BytesWritable(new byte[]{'>'}), ba.getWritableObject()); - initLazyObject(ba, new byte[]{'2','?','3'}, 0, 3); - assertEquals(new BytesWritable(new byte[]{'2','?','3'}), ba.getWritableObject()); + initLazyObject(ba, new byte[] {'%'}, 0, 1); + assertEquals(new BytesWritable(new byte[] {'%'}), ba.getWritableObject()); + initLazyObject(ba, new byte[] {'2', '>', '3'}, 1, 1); + assertEquals(new BytesWritable(new byte[] {'>'}), ba.getWritableObject()); + initLazyObject(ba, new byte[] {'2', '?', '3'}, 0, 3); + assertEquals(new BytesWritable(new byte[] {'2', '?', '3'}), ba.getWritableObject()); + initLazyObject(ba, new byte[] {'\n'}, 0, 1); + assertEquals(new BytesWritable(new byte[] {}), ba.getWritableObject()); } public void testLazyIntegerWrite() throws Throwable {