Index: serde/src/java/org/apache/hadoop/hive/serde2/lazy/ByteArrayRef.java
===================================================================
--- serde/src/java/org/apache/hadoop/hive/serde2/lazy/ByteArrayRef.java (revision 901519)
+++ serde/src/java/org/apache/hadoop/hive/serde2/lazy/ByteArrayRef.java (working copy)
@@ -20,14 +20,12 @@
/**
* ByteArrayRef stores a reference to a byte array.
*
- * The LazyObject hierarchy uses a reference to a single ByteArrayRef,
- * so that it's much faster to switch to the next row and release the
- * reference to the old row (so that the system can do garbage collection
- * if needed).
+ * The LazyObject hierarchy uses a reference to a single ByteArrayRef, so that
+ * it's much faster to switch to the next row and release the reference to the
+ * old row (so that the system can do garbage collection if needed).
*/
public class ByteArrayRef {
-
/**
* Stores the actual data.
*/
@@ -40,5 +38,5 @@
public void setData(byte[] data) {
this.data = data;
}
-
+
}
Index: serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyLong.java
===================================================================
--- serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyLong.java (revision 901519)
+++ serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyLong.java (working copy)
@@ -36,9 +36,9 @@
*
*
*/
-public class LazyLong extends LazyPrimitive {
+public class LazyLong extends
+ LazyPrimitive {
-
public LazyLong(LazyLongObjectInspector oi) {
super(oi);
data = new LongWritable();
@@ -48,7 +48,7 @@
super(copy);
data = new LongWritable(copy.data.get());
}
-
+
@Override
public void init(ByteArrayRef bytes, int start, int length) {
try {
@@ -67,38 +67,38 @@
* @param bytes
* @param start
* @param length
- * a UTF-8 encoded string representation of a long quantity.
+ * a UTF-8 encoded string representation of a long quantity.
* @return long the value represented by the argument
* @exception NumberFormatException
- * if the argument could not be parsed as a long quantity.
+ * if the argument could not be parsed as a long quantity.
*/
- public static long parseLong(byte[] bytes, int start, int length) throws NumberFormatException {
+ public static long parseLong(byte[] bytes, int start, int length)
+ throws NumberFormatException {
return parseLong(bytes, start, length, 10);
}
/**
* Parses the string argument as if it was an long value and returns the
* result. Throws NumberFormatException if the string does not represent an
- * long quantity. The second argument specifies the radix to use when
- * parsing the value.
+ * long quantity. The second argument specifies the radix to use when parsing
+ * the value.
*
* @param bytes
* @param start
* @param length
- * a UTF-8 encoded string representation of a long quantity.
+ * a UTF-8 encoded string representation of a long quantity.
* @param radix
- * the base to use for conversion.
+ * the base to use for conversion.
* @return the value represented by the argument
* @exception NumberFormatException
- * if the argument could not be parsed as an long quantity.
+ * if the argument could not be parsed as an long quantity.
*/
public static long parseLong(byte[] bytes, int start, int length, int radix)
throws NumberFormatException {
if (bytes == null) {
throw new NumberFormatException("String is null");
}
- if (radix < Character.MIN_RADIX ||
- radix > Character.MAX_RADIX) {
+ if (radix < Character.MIN_RADIX || radix > Character.MAX_RADIX) {
throw new NumberFormatException("Invalid radix: " + radix);
}
if (length == 0) {
@@ -107,9 +107,10 @@
int offset = start;
boolean negative = bytes[start] == '-';
if (negative || bytes[start] == '+') {
- offset ++;
+ offset++;
if (length == 1) {
- throw new NumberFormatException(LazyUtils.convertToString(bytes, start, length));
+ throw new NumberFormatException(LazyUtils.convertToString(bytes, start,
+ length));
}
}
@@ -117,61 +118,65 @@
}
/**
- /**
- * Parses the string argument as if it was an long value and returns the
+ * /** Parses the string argument as if it was an long value and returns the
* result. Throws NumberFormatException if the string does not represent an
- * long quantity. The second argument specifies the radix to use when
- * parsing the value.
+ * long quantity. The second argument specifies the radix to use when parsing
+ * the value.
*
* @param bytes
* @param start
* @param length
- * a UTF-8 encoded string representation of a long quantity.
+ * a UTF-8 encoded string representation of a long quantity.
* @param offset
- * the starting position after the sign (if exists)
+ * the starting position after the sign (if exists)
* @param radix
- * the base to use for conversion.
+ * the base to use for conversion.
* @param negative
- * whether the number is negative.
+ * whether the number is negative.
* @return the value represented by the argument
* @exception NumberFormatException
- * if the argument could not be parsed as an long quantity.
+ * if the argument could not be parsed as an long quantity.
*/
- private static long parse(byte[] bytes, int start, int length, int offset, int radix,
- boolean negative) {
+ private static long parse(byte[] bytes, int start, int length, int offset,
+ int radix, boolean negative) {
long max = Long.MIN_VALUE / radix;
long result = 0, end = start + length;
while (offset < end) {
int digit = LazyUtils.digit(bytes[offset++], radix);
if (digit == -1 || max > result) {
- throw new NumberFormatException(LazyUtils.convertToString(bytes, start, length));
+ throw new NumberFormatException(LazyUtils.convertToString(bytes, start,
+ length));
}
long next = result * radix - digit;
if (next > result) {
- throw new NumberFormatException(LazyUtils.convertToString(bytes, start, length));
+ throw new NumberFormatException(LazyUtils.convertToString(bytes, start,
+ length));
}
result = next;
}
if (!negative) {
result = -result;
if (result < 0) {
- throw new NumberFormatException(LazyUtils.convertToString(bytes, start, length));
+ throw new NumberFormatException(LazyUtils.convertToString(bytes, start,
+ length));
}
}
return result;
}
/**
- * Writes out the text representation of an integer using base 10 to an
+ * Writes out the text representation of an integer using base 10 to an
* OutputStream in UTF-8 encoding.
- *
- * Note: division by a constant (like 10) is much faster than division by
- * a variable. That's one of the reasons that we don't make radix a
- * parameter here.
- *
- * @param out the outputstream to write to
- * @param i an int to write out
- * @throws IOException
+ *
+ * Note: division by a constant (like 10) is much faster than division by a
+ * variable. That's one of the reasons that we don't make radix a parameter
+ * here.
+ *
+ * @param out
+ * the outputstream to write to
+ * @param i
+ * an int to write out
+ * @throws IOException
*/
public static void writeUTF8(OutputStream out, long i) throws IOException {
if (i == 0) {
@@ -187,14 +192,14 @@
// of overflow here.
i = -i;
}
-
+
long start = 1000000000000000000L;
- while (i/start == 0) {
+ while (i / start == 0) {
start /= 10;
}
-
+
while (start > 0) {
- out.write('0' - (int)((i / start) % 10));
+ out.write('0' - (int) ((i / start) % 10));
start /= 10;
}
}
Index: serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyShort.java
===================================================================
--- serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyShort.java (revision 901519)
+++ serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyShort.java (working copy)
@@ -33,18 +33,19 @@
*
*
*/
-public class LazyShort extends LazyPrimitive {
+public class LazyShort extends
+ LazyPrimitive {
public LazyShort(LazyShortObjectInspector oi) {
super(oi);
data = new ShortWritable();
}
-
+
public LazyShort(LazyShort copy) {
super(copy);
data = new ShortWritable(copy.data.get());
}
-
+
@Override
public void init(ByteArrayRef bytes, int start, int length) {
try {
@@ -63,12 +64,12 @@
* @param bytes
* @param start
* @param length
- * a UTF-8 encoded string representation of a short quantity.
+ * a UTF-8 encoded string representation of a short quantity.
* @return short the value represented by the argument
* @exception NumberFormatException
- * if the argument could not be parsed as a short quantity.
+ * if the argument could not be parsed as a short quantity.
*/
- public static short parseShort(byte[] bytes, int start, int length)
+ public static short parseShort(byte[] bytes, int start, int length)
throws NumberFormatException {
return parseShort(bytes, start, length, 10);
}
@@ -76,18 +77,18 @@
/**
* Parses the string argument as if it was a short value and returns the
* result. Throws NumberFormatException if the string does not represent a
- * single short quantity. The second argument specifies the radix to use
- * when parsing the value.
+ * single short quantity. The second argument specifies the radix to use when
+ * parsing the value.
*
* @param bytes
* @param start
* @param length
- * a UTF-8 encoded string representation of a short quantity.
+ * a UTF-8 encoded string representation of a short quantity.
* @param radix
- * the radix to use when parsing.
+ * the radix to use when parsing.
* @return short the value represented by the argument
* @exception NumberFormatException
- * if the argument could not be parsed as a short quantity.
+ * if the argument could not be parsed as a short quantity.
*/
public static short parseShort(byte[] bytes, int start, int length, int radix)
throws NumberFormatException {
Index: serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyUtils.java
===================================================================
--- serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyUtils.java (revision 901519)
+++ serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyUtils.java (working copy)
@@ -37,32 +37,38 @@
/**
* Returns the digit represented by character b.
- * @param b The ascii code of the character
- * @param radix The radix
- * @return -1 if it's invalid
+ *
+ * @param b
+ * The ascii code of the character
+ * @param radix
+ * The radix
+ * @return -1 if it's invalid
*/
public static int digit(int b, int radix) {
int r = -1;
- if (b >= '0' && b<='9') {
+ if (b >= '0' && b <= '9') {
r = b - '0';
- } else if (b >= 'A' && b<='Z') {
+ } else if (b >= 'A' && b <= 'Z') {
r = b - 'A' + 10;
} else if (b >= 'a' && b <= 'z') {
r = b - 'a' + 10;
}
- if (r >= radix) r = -1;
+ if (r >= radix) {
+ r = -1;
+ }
return r;
}
-
+
/**
- * Returns -1 if the first byte sequence is lexicographically less than the second;
- * returns +1 if the second byte sequence is lexicographically less than the first;
- * otherwise return 0.
+ * Returns -1 if the first byte sequence is lexicographically less than the
+ * second; returns +1 if the second byte sequence is lexicographically less
+ * than the first; otherwise return 0.
*/
- public static int compare(byte[] b1, int start1, int length1, byte[] b2, int start2, int length2) {
-
+ public static int compare(byte[] b1, int start1, int length1, byte[] b2,
+ int start2, int length2) {
+
int min = Math.min(length1, length2);
-
+
for (int i = 0; i < min; i++) {
if (b1[start1 + i] == b2[start2 + i]) {
continue;
@@ -71,19 +77,27 @@
return -1;
} else {
return 1;
- }
+ }
}
-
- if (length1 < length2) return -1;
- if (length1 > length2) return 1;
+
+ if (length1 < length2) {
+ return -1;
+ }
+ if (length1 > length2) {
+ return 1;
+ }
return 0;
}
-
+
/**
* Convert a UTF-8 byte array to String.
- * @param bytes The byte[] containing the UTF-8 String.
- * @param start The start position inside the bytes.
- * @param length The length of the data, starting from "start"
+ *
+ * @param bytes
+ * The byte[] containing the UTF-8 String.
+ * @param start
+ * The start position inside the bytes.
+ * @param length
+ * The length of the data, starting from "start"
* @return The unicode String
*/
public static String convertToString(byte[] bytes, int start, int length) {
@@ -94,19 +108,22 @@
}
}
+ static byte[] trueBytes = { (byte) 't', 'r', 'u', 'e' };
+ static byte[] falseBytes = { (byte) 'f', 'a', 'l', 's', 'e' };
- static byte[] trueBytes = {(byte)'t', 'r', 'u', 'e'};
- static byte[] falseBytes = {(byte)'f', 'a', 'l', 's', 'e'};
-
/**
* Write the bytes with special characters escaped.
- * @param escaped Whether the data should be written out in an escaped way.
- * @param escapeChar if escaped, the char for prefixing special characters.
- * @param needsEscape if escaped, whether a specific character needs escaping.
- * This array should have size of 128.
+ *
+ * @param escaped
+ * Whether the data should be written out in an escaped way.
+ * @param escapeChar
+ * if escaped, the char for prefixing special characters.
+ * @param needsEscape
+ * if escaped, whether a specific character needs escaping. This
+ * array should have size of 128.
*/
- private static void writeEscaped(OutputStream out, byte[] bytes,
- int start, int len, boolean escaped, byte escapeChar, boolean[] needsEscape)
+ private static void writeEscaped(OutputStream out, byte[] bytes, int start,
+ int len, boolean escaped, byte escapeChar, boolean[] needsEscape)
throws IOException {
if (escaped) {
int end = start + len;
@@ -126,71 +143,79 @@
out.write(bytes, 0, len);
}
}
-
-
+
/**
- * Write out the text representation of a Primitive Object to a UTF8 byte stream.
- * @param out The UTF8 byte OutputStream
- * @param o The primitive Object
- * @param needsEscape Whether a character needs escaping. This array should have size of 128.
+ * Write out the text representation of a Primitive Object to a UTF8 byte
+ * stream.
+ *
+ * @param out
+ * The UTF8 byte OutputStream
+ * @param o
+ * The primitive Object
+ * @param needsEscape
+ * Whether a character needs escaping. This array should have size of
+ * 128.
*/
- public static void writePrimitiveUTF8(OutputStream out, Object o, PrimitiveObjectInspector oi,
- boolean escaped, byte escapeChar, boolean[] needsEscape) throws IOException {
-
+ public static void writePrimitiveUTF8(OutputStream out, Object o,
+ PrimitiveObjectInspector oi, boolean escaped, byte escapeChar,
+ boolean[] needsEscape) throws IOException {
+
switch (oi.getPrimitiveCategory()) {
- case BOOLEAN: {
- boolean b = ((BooleanObjectInspector)oi).get(o);
- if (b) {
- out.write(trueBytes, 0, trueBytes.length);
- } else {
- out.write(falseBytes, 0, falseBytes.length);
- }
- break;
+ case BOOLEAN: {
+ boolean b = ((BooleanObjectInspector) oi).get(o);
+ if (b) {
+ out.write(trueBytes, 0, trueBytes.length);
+ } else {
+ out.write(falseBytes, 0, falseBytes.length);
}
- case BYTE: {
- LazyInteger.writeUTF8(out, ((ByteObjectInspector)oi).get(o));
- break;
- }
- case SHORT: {
- LazyInteger.writeUTF8(out, ((ShortObjectInspector)oi).get(o));
- break;
- }
- case INT: {
- LazyInteger.writeUTF8(out, ((IntObjectInspector)oi).get(o));
- break;
- }
- case LONG: {
- LazyLong.writeUTF8(out, ((LongObjectInspector)oi).get(o));
- break;
- }
- case FLOAT: {
- float f = ((FloatObjectInspector)oi).get(o);
- ByteBuffer b = Text.encode(String.valueOf(f));
- out.write(b.array(), 0, b.limit());
- break;
- }
- case DOUBLE: {
- double d = ((DoubleObjectInspector)oi).get(o);
- ByteBuffer b = Text.encode(String.valueOf(d));
- out.write(b.array(), 0, b.limit());
- break;
- }
- case STRING: {
- Text t = ((StringObjectInspector)oi).getPrimitiveWritableObject(o);
- writeEscaped(out, t.getBytes(), 0, t.getLength(), escaped, escapeChar, needsEscape);
- break;
- }
- default: {
- throw new RuntimeException("Hive internal error.");
- }
+ break;
}
+ case BYTE: {
+ LazyInteger.writeUTF8(out, ((ByteObjectInspector) oi).get(o));
+ break;
+ }
+ case SHORT: {
+ LazyInteger.writeUTF8(out, ((ShortObjectInspector) oi).get(o));
+ break;
+ }
+ case INT: {
+ LazyInteger.writeUTF8(out, ((IntObjectInspector) oi).get(o));
+ break;
+ }
+ case LONG: {
+ LazyLong.writeUTF8(out, ((LongObjectInspector) oi).get(o));
+ break;
+ }
+ case FLOAT: {
+ float f = ((FloatObjectInspector) oi).get(o);
+ ByteBuffer b = Text.encode(String.valueOf(f));
+ out.write(b.array(), 0, b.limit());
+ break;
+ }
+ case DOUBLE: {
+ double d = ((DoubleObjectInspector) oi).get(o);
+ ByteBuffer b = Text.encode(String.valueOf(d));
+ out.write(b.array(), 0, b.limit());
+ break;
+ }
+ case STRING: {
+ Text t = ((StringObjectInspector) oi).getPrimitiveWritableObject(o);
+ writeEscaped(out, t.getBytes(), 0, t.getLength(), escaped, escapeChar,
+ needsEscape);
+ break;
+ }
+ default: {
+ throw new RuntimeException("Hive internal error.");
+ }
+ }
}
-
+
public static int hashBytes(byte[] data, int start, int len) {
int hash = 1;
- for (int i = start; i < len; i++)
- hash = (31 * hash) + (int)data[i];
+ for (int i = start; i < len; i++) {
+ hash = (31 * hash) + data[i];
+ }
return hash;
}
-
+
}
Index: serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyByte.java
===================================================================
--- serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyByte.java (revision 901519)
+++ serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyByte.java (working copy)
@@ -33,7 +33,8 @@
*
*
*/
-public class LazyByte extends LazyPrimitive {
+public class LazyByte extends
+ LazyPrimitive {
public LazyByte(LazyByteObjectInspector oi) {
super(oi);
@@ -44,7 +45,7 @@
super(copy);
data = new ByteWritable(copy.data.get());
}
-
+
@Override
public void init(ByteArrayRef bytes, int start, int length) {
try {
@@ -54,7 +55,7 @@
isNull = true;
}
}
-
+
/**
* Parses the string argument as if it was a byte value and returns the
* result. Throws NumberFormatException if the string does not represent a
@@ -63,15 +64,16 @@
* @param bytes
* @param start
* @param length
- * a UTF-8 encoded string representation of a single byte quantity.
+ * a UTF-8 encoded string representation of a single byte quantity.
* @return byte the value represented by the argument
* @throws NumberFormatException
- * if the argument could not be parsed as a byte quantity.
+ * if the argument could not be parsed as a byte quantity.
*/
- public static byte parseByte(byte[] bytes, int start, int length) throws NumberFormatException {
- return parseByte(bytes, start, length, 10);
+ public static byte parseByte(byte[] bytes, int start, int length)
+ throws NumberFormatException {
+ return parseByte(bytes, start, length, 10);
}
-
+
/**
* Parses the string argument as if it was a byte value and returns the
* result. Throws NumberFormatException if the string does not represent a
@@ -81,22 +83,21 @@
* @param bytes
* @param start
* @param length
- * a UTF-8 encoded string representation of a single byte quantity.
+ * a UTF-8 encoded string representation of a single byte quantity.
* @param radix
- * the radix to use when parsing.
+ * the radix to use when parsing.
* @return byte the value represented by the argument
* @throws NumberFormatException
- * if the argument could not be parsed as a byte quantity.
+ * if the argument could not be parsed as a byte quantity.
*/
public static byte parseByte(byte[] bytes, int start, int length, int radix)
- throws NumberFormatException {
- int intValue = LazyInteger.parseInt(bytes, start, length, radix);
- byte result = (byte) intValue;
- if (result == intValue) {
- return result;
- }
- throw new NumberFormatException();
+ throws NumberFormatException {
+ int intValue = LazyInteger.parseInt(bytes, start, length, radix);
+ byte result = (byte) intValue;
+ if (result == intValue) {
+ return result;
+ }
+ throw new NumberFormatException();
}
-
}
Index: serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyFloat.java
===================================================================
--- serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyFloat.java (revision 901519)
+++ serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyFloat.java (working copy)
@@ -27,18 +27,19 @@
* LazyObject for storing a value of Double.
*
*/
-public class LazyFloat extends LazyPrimitive {
+public class LazyFloat extends
+ LazyPrimitive {
public LazyFloat(LazyFloatObjectInspector oi) {
super(oi);
data = new FloatWritable();
}
-
+
public LazyFloat(LazyFloat copy) {
super(copy);
data = new FloatWritable(copy.data.get());
- }
-
+ }
+
@Override
public void init(ByteArrayRef bytes, int start, int length) {
try {
Index: serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyObject.java
===================================================================
--- serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyObject.java (revision 901519)
+++ serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyObject.java (working copy)
@@ -22,40 +22,45 @@
/**
* LazyObject stores an object in a range of bytes in a byte[].
*
- * A LazyObject can represent any primitive object or hierarchical object
- * like array, map or struct.
+ * A LazyObject can represent any primitive object or hierarchical object like
+ * array, map or struct.
*/
public abstract class LazyObject {
OI oi;
-
+
/**
* Create a LazyObject.
- * @param oi Derived classes can access meta information about this Lazy
- * Object (e.g, separator, nullSequence, escaper) from it.
+ *
+ * @param oi
+ * Derived classes can access meta information about this Lazy Object
+ * (e.g, separator, nullSequence, escaper) from it.
*/
protected LazyObject(OI oi) {
this.oi = oi;
}
-
+
/**
- * Set the data for this LazyObject.
- * We take ByteArrayRef instead of byte[] so that we will be able to drop
- * the reference to byte[] by a single assignment.
- * The ByteArrayRef object can be reused across multiple rows.
- * @param bytes The wrapper of the byte[].
- * @param start The start position inside the bytes.
- * @param length The length of the data, starting from "start"
+ * Set the data for this LazyObject. We take ByteArrayRef instead of byte[] so
+ * that we will be able to drop the reference to byte[] by a single
+ * assignment. The ByteArrayRef object can be reused across multiple rows.
+ *
+ * @param bytes
+ * The wrapper of the byte[].
+ * @param start
+ * The start position inside the bytes.
+ * @param length
+ * The length of the data, starting from "start"
* @see ByteArrayRef
*/
public abstract void init(ByteArrayRef bytes, int start, int length);
/**
- * If the LazyObject is a primitive Object, then deserialize it and return
- * the actual primitive Object.
- * Otherwise (array, map, struct), return this.
+ * If the LazyObject is a primitive Object, then deserialize it and return the
+ * actual primitive Object. Otherwise (array, map, struct), return this.
*/
public abstract Object getObject();
-
+
+ @Override
public abstract int hashCode();
}
Index: serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyString.java
===================================================================
--- serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyString.java (revision 901519)
+++ serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyString.java (working copy)
@@ -34,24 +34,24 @@
super(copy);
data = new Text(copy.data);
}
-
+
@Override
public void init(ByteArrayRef bytes, int start, int length) {
if (oi.isEscaped()) {
byte escapeChar = oi.getEscapeChar();
byte[] inputBytes = bytes.getData();
-
+
// First calculate the length of the output string
int outputLength = 0;
- for (int i=0; i createLazyPrimitiveClass(PrimitiveObjectInspector oi) {
+ public static LazyPrimitive, ?> createLazyPrimitiveClass(
+ PrimitiveObjectInspector oi) {
PrimitiveCategory p = oi.getPrimitiveCategory();
- switch(p) {
- case BOOLEAN: {
- return new LazyBoolean((LazyBooleanObjectInspector)oi);
- }
- case BYTE: {
- return new LazyByte((LazyByteObjectInspector)oi);
- }
- case SHORT: {
- return new LazyShort((LazyShortObjectInspector)oi);
- }
- case INT: {
- return new LazyInteger((LazyIntObjectInspector)oi);
- }
- case LONG: {
- return new LazyLong((LazyLongObjectInspector)oi);
- }
- case FLOAT: {
- return new LazyFloat((LazyFloatObjectInspector)oi);
- }
- case DOUBLE: {
- return new LazyDouble((LazyDoubleObjectInspector)oi);
- }
- case STRING: {
- return new LazyString((LazyStringObjectInspector)oi);
- }
- default: {
- throw new RuntimeException("Internal error: no LazyObject for " + p);
- }
+ switch (p) {
+ case BOOLEAN: {
+ return new LazyBoolean((LazyBooleanObjectInspector) oi);
}
+ case BYTE: {
+ return new LazyByte((LazyByteObjectInspector) oi);
+ }
+ case SHORT: {
+ return new LazyShort((LazyShortObjectInspector) oi);
+ }
+ case INT: {
+ return new LazyInteger((LazyIntObjectInspector) oi);
+ }
+ case LONG: {
+ return new LazyLong((LazyLongObjectInspector) oi);
+ }
+ case FLOAT: {
+ return new LazyFloat((LazyFloatObjectInspector) oi);
+ }
+ case DOUBLE: {
+ return new LazyDouble((LazyDoubleObjectInspector) oi);
+ }
+ case STRING: {
+ return new LazyString((LazyStringObjectInspector) oi);
+ }
+ default: {
+ throw new RuntimeException("Internal error: no LazyObject for " + p);
+ }
+ }
}
/**
@@ -96,70 +88,75 @@
*/
public static LazyObject createLazyObject(ObjectInspector oi) {
ObjectInspector.Category c = oi.getCategory();
- switch(c) {
+ switch (c) {
case PRIMITIVE:
- return createLazyPrimitiveClass((PrimitiveObjectInspector)oi);
+ return createLazyPrimitiveClass((PrimitiveObjectInspector) oi);
case MAP:
- return new LazyMap((LazyMapObjectInspector)oi);
- case LIST:
- return new LazyArray((LazyListObjectInspector)oi);
+ return new LazyMap((LazyMapObjectInspector) oi);
+ case LIST:
+ return new LazyArray((LazyListObjectInspector) oi);
case STRUCT:
- return new LazyStruct((LazySimpleStructObjectInspector)oi);
+ return new LazyStruct((LazySimpleStructObjectInspector) oi);
}
throw new RuntimeException("Hive LazySerDe Internal error.");
}
-
+
/**
* Create a hierarchical ObjectInspector for LazyObject with the given
* typeInfo.
- * @param typeInfo The type information for the LazyObject
- * @param separator The array of separators for delimiting each level
- * @param separatorIndex The current level (for separators). List(array),
- * struct uses 1 level of separator, and map uses 2
- * levels: the first one for delimiting entries, the
- * second one for delimiting key and values.
- * @param nullSequence The sequence of bytes representing NULL.
- * @return The ObjectInspector
+ *
+ * @param typeInfo
+ * The type information for the LazyObject
+ * @param separator
+ * The array of separators for delimiting each level
+ * @param separatorIndex
+ * The current level (for separators). List(array), struct uses 1
+ * level of separator, and map uses 2 levels: the first one for
+ * delimiting entries, the second one for delimiting key and values.
+ * @param nullSequence
+ * The sequence of bytes representing NULL.
+ * @return The ObjectInspector
*/
- public static ObjectInspector createLazyObjectInspector(TypeInfo typeInfo, byte[] separator,
- int separatorIndex, Text nullSequence, boolean escaped, byte escapeChar) {
+ public static ObjectInspector createLazyObjectInspector(TypeInfo typeInfo,
+ byte[] separator, int separatorIndex, Text nullSequence, boolean escaped,
+ byte escapeChar) {
ObjectInspector.Category c = typeInfo.getCategory();
- switch(c) {
+ switch (c) {
case PRIMITIVE:
return LazyPrimitiveObjectInspectorFactory.getLazyObjectInspector(
- ((PrimitiveTypeInfo)typeInfo).getPrimitiveCategory(), escaped, escapeChar);
+ ((PrimitiveTypeInfo) typeInfo).getPrimitiveCategory(), escaped,
+ escapeChar);
case MAP:
return LazyObjectInspectorFactory.getLazySimpleMapObjectInspector(
- createLazyObjectInspector(((MapTypeInfo)typeInfo).getMapKeyTypeInfo(),
- separator, separatorIndex+2, nullSequence, escaped, escapeChar),
- createLazyObjectInspector(((MapTypeInfo)typeInfo).getMapValueTypeInfo(),
- separator, separatorIndex+2, nullSequence, escaped, escapeChar),
- separator[separatorIndex],
- separator[separatorIndex+1],
+ createLazyObjectInspector(((MapTypeInfo) typeInfo)
+ .getMapKeyTypeInfo(), separator, separatorIndex + 2,
+ nullSequence, escaped, escapeChar), createLazyObjectInspector(
+ ((MapTypeInfo) typeInfo).getMapValueTypeInfo(), separator,
+ separatorIndex + 2, nullSequence, escaped, escapeChar),
+ separator[separatorIndex], separator[separatorIndex + 1],
nullSequence, escaped, escapeChar);
- case LIST:
+ case LIST:
return LazyObjectInspectorFactory.getLazySimpleListObjectInspector(
- createLazyObjectInspector(((ListTypeInfo)typeInfo).getListElementTypeInfo(),
- separator, separatorIndex+1, nullSequence, escaped, escapeChar),
- separator[separatorIndex],
+ createLazyObjectInspector(((ListTypeInfo) typeInfo)
+ .getListElementTypeInfo(), separator, separatorIndex + 1,
+ nullSequence, escaped, escapeChar), separator[separatorIndex],
nullSequence, escaped, escapeChar);
case STRUCT:
- StructTypeInfo structTypeInfo = (StructTypeInfo)typeInfo;
+ StructTypeInfo structTypeInfo = (StructTypeInfo) typeInfo;
List fieldNames = structTypeInfo.getAllStructFieldNames();
- List fieldTypeInfos = structTypeInfo.getAllStructFieldTypeInfos();
- List fieldObjectInspectors = new ArrayList(fieldTypeInfos.size());
- for(int i=0; i fieldTypeInfos = structTypeInfo
+ .getAllStructFieldTypeInfos();
+ List fieldObjectInspectors = new ArrayList(
+ fieldTypeInfos.size());
+ for (int i = 0; i < fieldTypeInfos.size(); i++) {
+ fieldObjectInspectors.add(createLazyObjectInspector(fieldTypeInfos
+ .get(i), separator, separatorIndex + 1, nullSequence, escaped,
+ escapeChar));
}
return LazyObjectInspectorFactory.getLazySimpleStructObjectInspector(
- fieldNames,
- fieldObjectInspectors,
- separator[separatorIndex],
- nullSequence,
- false, escaped, escapeChar);
+ fieldNames, fieldObjectInspectors, separator[separatorIndex],
+ nullSequence, false, escaped, escapeChar);
}
throw new RuntimeException("Hive LazySerDe Internal error.");
@@ -169,29 +166,33 @@
* Create a hierarchical ObjectInspector for LazyStruct with the given
* columnNames and columnTypeInfos.
*
- * @param lastColumnTakesRest whether the last column of the struct should take
- * the rest of the row if there are extra fields.
- * @see LazyFactory#createLazyObjectInspector(TypeInfo, byte[], int, Text, boolean, byte)
- */
- public static ObjectInspector createLazyStructInspector(List columnNames,
- List typeInfos, byte[] separators,
- Text nullSequence, boolean lastColumnTakesRest, boolean escaped, byte escapeChar) {
- ArrayList columnObjectInspectors =
- new ArrayList(typeInfos.size());
- for (int i=0; i columnNames, List typeInfos, byte[] separators,
+ Text nullSequence, boolean lastColumnTakesRest, boolean escaped,
+ byte escapeChar) {
+ ArrayList columnObjectInspectors = new ArrayList(
+ typeInfos.size());
+ for (int i = 0; i < typeInfos.size(); i++) {
+ columnObjectInspectors.add(LazyFactory.createLazyObjectInspector(
+ typeInfos.get(i), separators, 1, nullSequence, escaped, escapeChar));
}
- return
- LazyObjectInspectorFactory.getLazySimpleStructObjectInspector(columnNames,
- columnObjectInspectors, separators[0], nullSequence, lastColumnTakesRest, escaped, escapeChar);
+ return LazyObjectInspectorFactory.getLazySimpleStructObjectInspector(
+ columnNames, columnObjectInspectors, separators[0], nullSequence,
+ lastColumnTakesRest, escaped, escapeChar);
}
-
+
/**
* Create a hierarchical ObjectInspector for ColumnarStruct with the given
* columnNames and columnTypeInfos.
*
- * @see LazyFactory#createLazyObjectInspector(TypeInfo, byte[], int, Text, boolean, byte)
+ * @see LazyFactory#createLazyObjectInspector(TypeInfo, byte[], int, Text,
+ * boolean, byte)
*/
public static ObjectInspector createColumnarStructInspector(
List columnNames, List columnTypes, byte[] separators,
@@ -199,11 +200,12 @@
ArrayList columnObjectInspectors = new ArrayList(
columnTypes.size());
for (int i = 0; i < columnTypes.size(); i++) {
- columnObjectInspectors.add(LazyFactory.createLazyObjectInspector(
- columnTypes.get(i), separators, 1, nullSequence, escaped, escapeChar));
+ columnObjectInspectors
+ .add(LazyFactory.createLazyObjectInspector(columnTypes.get(i),
+ separators, 1, nullSequence, escaped, escapeChar));
}
return ObjectInspectorFactory.getColumnarStructObjectInspector(columnNames,
columnObjectInspectors, nullSequence);
}
-
+
}
Index: serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyPrimitive.java
===================================================================
--- serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyPrimitive.java (revision 901519)
+++ serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyPrimitive.java (working copy)
@@ -23,8 +23,8 @@
/**
* LazyPrimitive stores a primitive Object in a LazyObject.
*/
-public abstract class LazyPrimitive extends LazyObject {
+public abstract class LazyPrimitive
+ extends LazyObject {
LazyPrimitive(OI oi) {
super(oi);
@@ -39,9 +39,10 @@
boolean isNull = false;
/**
- * Returns the primitive object represented by this LazyObject.
- * This is useful because it can make sure we have "null" for null objects.
+ * Returns the primitive object represented by this LazyObject. This is useful
+ * because it can make sure we have "null" for null objects.
*/
+ @Override
public Object getObject() {
return isNull ? null : this;
}
@@ -49,13 +50,15 @@
public T getWritableObject() {
return isNull ? null : data;
}
-
+
+ @Override
public String toString() {
return isNull ? null : data.toString();
}
-
- public int hashCode(){
+
+ @Override
+ public int hashCode() {
return isNull ? 0 : data.hashCode();
}
-
+
}
Index: serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyDouble.java
===================================================================
--- serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyDouble.java (revision 901519)
+++ serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyDouble.java (working copy)
@@ -27,7 +27,8 @@
* LazyObject for storing a value of Double.
*
*/
-public class LazyDouble extends LazyPrimitive {
+public class LazyDouble extends
+ LazyPrimitive {
public LazyDouble(LazyDoubleObjectInspector oi) {
super(oi);
@@ -37,8 +38,8 @@
public LazyDouble(LazyDouble copy) {
super(copy);
data = new DoubleWritable(copy.data.get());
- }
-
+ }
+
@Override
public void init(ByteArrayRef bytes, int start, int length) {
try {
Index: serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySimpleSerDe.java
===================================================================
--- serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySimpleSerDe.java (revision 901519)
+++ serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySimpleSerDe.java (working copy)
@@ -50,25 +50,26 @@
import org.apache.hadoop.io.Writable;
/**
- * LazySimpleSerDe can be used to read the same data format as
+ * LazySimpleSerDe can be used to read the same data format as
* MetadataTypedColumnsetSerDe and TCTLSeparatedProtocol.
*
- * However, LazySimpleSerDe creates Objects in a lazy way, to
- * provide better performance.
+ * However, LazySimpleSerDe creates Objects in a lazy way, to provide better
+ * performance.
*
- * Also LazySimpleSerDe outputs typed columns instead of treating
- * all columns as String like MetadataTypedColumnsetSerDe.
+ * Also LazySimpleSerDe outputs typed columns instead of treating all columns as
+ * String like MetadataTypedColumnsetSerDe.
*/
public class LazySimpleSerDe implements SerDe {
- public static final Log LOG = LogFactory.getLog(
- LazySimpleSerDe.class.getName());
+ public static final Log LOG = LogFactory.getLog(LazySimpleSerDe.class
+ .getName());
- final public static byte[] DefaultSeparators = {(byte)1, (byte)2, (byte)3};
+ final public static byte[] DefaultSeparators = { (byte) 1, (byte) 2, (byte) 3 };
private ObjectInspector cachedObjectInspector;
- private boolean useJSONSerialize; // use json to serialize
+ private boolean useJSONSerialize; // use json to serialize
+ @Override
public String toString() {
return getClass().toString()
+ "["
@@ -85,16 +86,19 @@
/**
* Return the byte value of the number string.
- * @param altValue The string containing a number.
- * @param defaultVal If the altValue does not represent a number,
- * return the defaultVal.
+ *
+ * @param altValue
+ * The string containing a number.
+ * @param defaultVal
+ * If the altValue does not represent a number, return the
+ * defaultVal.
*/
public static byte getByte(String altValue, byte defaultVal) {
if (altValue != null && altValue.length() > 0) {
try {
return Byte.valueOf(altValue).byteValue();
- } catch(NumberFormatException e) {
- return (byte)altValue.charAt(0);
+ } catch (NumberFormatException e) {
+ return (byte) altValue.charAt(0);
}
}
return defaultVal;
@@ -108,12 +112,12 @@
boolean lastColumnTakesRest;
List columnNames;
List columnTypes;
-
+
boolean escaped;
byte escapeChar;
boolean[] needsEscape;
boolean jsonSerialize;
-
+
public List getColumnTypes() {
return columnTypes;
}
@@ -121,7 +125,7 @@
public List getColumnNames() {
return columnNames;
}
-
+
public byte[] getSeparators() {
return separators;
}
@@ -141,13 +145,15 @@
public boolean isLastColumnTakesRest() {
return lastColumnTakesRest;
}
-
+
public boolean isEscaped() {
return escaped;
}
+
public byte getEscapeChar() {
return escapeChar;
}
+
public boolean[] getNeedsEscape() {
return needsEscape;
}
@@ -160,44 +166,45 @@
}
/**
- * @param jsonSerialize the jsonSerialize to set
+ * @param jsonSerialize
+ * the jsonSerialize to set
*/
public void setJsonSerialize(boolean jsonSerialize) {
this.jsonSerialize = jsonSerialize;
}
-
+
}
SerDeParameters serdeParams = null;
/**
- * Initialize the SerDe given the parameters.
- * serialization.format: separator char or byte code (only supports
- * byte-value up to 127)
- * columns: ","-separated column names
- * columns.types: ",", ":", or ";"-separated column types
- * @see SerDe#initialize(Configuration, Properties)
+ * Initialize the SerDe given the parameters. serialization.format: separator
+ * char or byte code (only supports byte-value up to 127) columns:
+ * ","-separated column names columns.types: ",", ":", or ";"-separated column
+ * types
+ *
+ * @see SerDe#initialize(Configuration, Properties)
*/
- public void initialize(Configuration job, Properties tbl)
- throws SerDeException {
+ public void initialize(Configuration job, Properties tbl)
+ throws SerDeException {
serdeParams = LazySimpleSerDe.initSerdeParams(job, tbl, getClass()
.getName());
// Create the ObjectInspectors for the fields
- cachedObjectInspector = LazyFactory.createLazyStructInspector(
- serdeParams.getColumnNames(), serdeParams.getColumnTypes(),
- serdeParams.getSeparators(), serdeParams.getNullSequence(),
- serdeParams.isLastColumnTakesRest(),
- serdeParams.isEscaped(),
- serdeParams.getEscapeChar());
+ cachedObjectInspector = LazyFactory.createLazyStructInspector(serdeParams
+ .getColumnNames(), serdeParams.getColumnTypes(), serdeParams
+ .getSeparators(), serdeParams.getNullSequence(), serdeParams
+ .isLastColumnTakesRest(), serdeParams.isEscaped(), serdeParams
+ .getEscapeChar());
- if (serdeParams.isJsonSerialize())
+ if (serdeParams.isJsonSerialize()) {
setUseJSONSerialize(true);
-
+ }
+
cachedLazyStruct = (LazyStruct) LazyFactory
- .createLazyObject(cachedObjectInspector);
-
+ .createLazyObject(cachedObjectInspector);
+
LOG.debug("LazySimpleSerDe initialized with: columnNames="
+ serdeParams.columnNames + " columnTypes=" + serdeParams.columnTypes
+ " separator=" + Arrays.asList(serdeParams.separators)
@@ -232,8 +239,9 @@
.equalsIgnoreCase("true"));
String useJsonSerialize = tbl
- .getProperty(Constants.SERIALIZATION_USE_JSON_OBJECTS);
- serdeParams.jsonSerialize = (useJsonSerialize != null && useJsonSerialize.equalsIgnoreCase("true"));
+ .getProperty(Constants.SERIALIZATION_USE_JSON_OBJECTS);
+ serdeParams.jsonSerialize = (useJsonSerialize != null && useJsonSerialize
+ .equalsIgnoreCase("true"));
// Read the configuration parameters
String columnNameProperty = tbl.getProperty(Constants.LIST_COLUMNS);
@@ -251,8 +259,9 @@
// Default type: all string
StringBuilder sb = new StringBuilder();
for (int i = 0; i < serdeParams.columnNames.size(); i++) {
- if (i > 0)
+ if (i > 0) {
sb.append(":");
+ }
sb.append(Constants.STRING_TYPE_NAME);
}
columnTypeProperty = sb.toString();
@@ -271,37 +280,39 @@
// Create the LazyObject for storing the rows
serdeParams.rowTypeInfo = TypeInfoFactory.getStructTypeInfo(
serdeParams.columnNames, serdeParams.columnTypes);
-
+
// Get the escape information
String escapeProperty = tbl.getProperty(Constants.ESCAPE_CHAR);
serdeParams.escaped = (escapeProperty != null);
if (serdeParams.escaped) {
- serdeParams.escapeChar = getByte(escapeProperty, (byte)'\\');
+ serdeParams.escapeChar = getByte(escapeProperty, (byte) '\\');
}
if (serdeParams.escaped) {
serdeParams.needsEscape = new boolean[128];
- for (int i=0; i<128; i++) {
+ for (int i = 0; i < 128; i++) {
serdeParams.needsEscape[i] = false;
}
serdeParams.needsEscape[serdeParams.escapeChar] = true;
- for (int i=0; i getSerializedClass() {
return Text.class;
}
-
+
Text serializeCache = new Text();
ByteStream.Output serializeStream = new ByteStream.Output();
+
/**
* Serialize a row of data.
- * @param obj The row object
- * @param objInspector The ObjectInspector for the row object
- * @return The serialized Writable object
- * @throws IOException
- * @see SerDe#serialize(Object, ObjectInspector)
+ *
+ * @param obj
+ * The row object
+ * @param objInspector
+ * The ObjectInspector for the row object
+ * @return The serialized Writable object
+ * @throws IOException
+ * @see SerDe#serialize(Object, ObjectInspector)
*/
- public Writable serialize(Object obj, ObjectInspector objInspector)
+ public Writable serialize(Object obj, ObjectInspector objInspector)
throws SerDeException {
if (objInspector.getCategory() != Category.STRUCT) {
- throw new SerDeException(getClass().toString()
- + " can only serialize struct types, but we got: "
+ throw new SerDeException(getClass().toString()
+ + " can only serialize struct types, but we got: "
+ objInspector.getTypeName());
}
// Prepare the field ObjectInspectors
- StructObjectInspector soi = (StructObjectInspector)objInspector;
+ StructObjectInspector soi = (StructObjectInspector) objInspector;
List extends StructField> fields = soi.getAllStructFieldRefs();
List list = soi.getStructFieldsDataAsList(obj);
- List extends StructField> declaredFields =(serdeParams.rowTypeInfo != null && ((StructTypeInfo) serdeParams.rowTypeInfo)
- .getAllStructFieldNames().size()>0)? ((StructObjectInspector)getObjectInspector())
+ List extends StructField> declaredFields = (serdeParams.rowTypeInfo != null && ((StructTypeInfo) serdeParams.rowTypeInfo)
+ .getAllStructFieldNames().size() > 0) ? ((StructObjectInspector) getObjectInspector())
.getAllStructFieldRefs()
: null;
@@ -372,9 +387,9 @@
try {
// Serialize each field
- for (int i=0; i0) {
+ if (i > 0) {
serializeStream.write(serdeParams.separators[0]);
}
// Get the field objectInspector and the field object.
@@ -382,23 +397,22 @@
Object f = (list == null ? null : list.get(i));
if (declaredFields != null && i >= declaredFields.size()) {
- throw new SerDeException(
- "Error: expecting " + declaredFields.size()
+ throw new SerDeException("Error: expecting " + declaredFields.size()
+ " but asking for field " + i + "\n" + "data=" + obj + "\n"
+ "tableType=" + serdeParams.rowTypeInfo.toString() + "\n"
- + "dataType="
+ + "dataType="
+ TypeInfoUtils.getTypeInfoFromObjectInspector(objInspector));
}
-
- // If the field that is passed in is NOT a primitive, and either the
- // field is not declared (no schema was given at initialization), or
- // the field is declared as a primitive in initialization, serialize
- // the data to JSON string. Otherwise serialize the data in the
+
+ // If the field that is passed in is NOT a primitive, and either the
+ // field is not declared (no schema was given at initialization), or
+ // the field is declared as a primitive in initialization, serialize
+ // the data to JSON string. Otherwise serialize the data in the
// delimited way.
if (!foi.getCategory().equals(Category.PRIMITIVE)
- && (declaredFields == null ||
- declaredFields.get(i).getFieldObjectInspector().getCategory()
- .equals(Category.PRIMITIVE) || useJSONSerialize)) {
+ && (declaredFields == null
+ || declaredFields.get(i).getFieldObjectInspector()
+ .getCategory().equals(Category.PRIMITIVE) || useJSONSerialize)) {
serialize(serializeStream, SerDeUtils.getJSONString(f, foi),
PrimitiveObjectInspectorFactory.javaStringObjectInspector,
serdeParams.separators, 1, serdeParams.nullSequence,
@@ -406,8 +420,8 @@
serdeParams.needsEscape);
} else {
serialize(serializeStream, f, foi, serdeParams.separators, 1,
- serdeParams.nullSequence, serdeParams.escaped, serdeParams.escapeChar,
- serdeParams.needsEscape);
+ serdeParams.nullSequence, serdeParams.escaped,
+ serdeParams.escapeChar, serdeParams.needsEscape);
}
}
} catch (IOException e) {
@@ -415,105 +429,119 @@
}
// TODO: The copy of data is unnecessary, but there is no work-around
// since we cannot directly set the private byte[] field inside Text.
- serializeCache.set(serializeStream.getData(), 0,
- serializeStream.getCount());
+ serializeCache
+ .set(serializeStream.getData(), 0, serializeStream.getCount());
return serializeCache;
}
/**
* Serialize the row into the StringBuilder.
- * @param out The StringBuilder to store the serialized data.
- * @param obj The object for the current field.
- * @param objInspector The ObjectInspector for the current Object.
- * @param separators The separators array.
- * @param level The current level of separator.
- * @param nullSequence The byte sequence representing the NULL value.
- * @param escaped Whether we need to escape the data when writing out
- * @param escapeChar Which char to use as the escape char, e.g. '\\'
- * @param needsEscape Which chars needs to be escaped. This array should have size of 128.
- * Negative byte values (or byte values >= 128) are never escaped.
- * @throws IOException
+ *
+ * @param out
+ * The StringBuilder to store the serialized data.
+ * @param obj
+ * The object for the current field.
+ * @param objInspector
+ * The ObjectInspector for the current Object.
+ * @param separators
+ * The separators array.
+ * @param level
+ * The current level of separator.
+ * @param nullSequence
+ * The byte sequence representing the NULL value.
+ * @param escaped
+ * Whether we need to escape the data when writing out
+ * @param escapeChar
+ * Which char to use as the escape char, e.g. '\\'
+ * @param needsEscape
+ * Which chars needs to be escaped. This array should have size of
+ * 128. Negative byte values (or byte values >= 128) are never
+ * escaped.
+ * @throws IOException
*/
- public static void serialize(ByteStream.Output out, Object obj,
+ public static void serialize(ByteStream.Output out, Object obj,
ObjectInspector objInspector, byte[] separators, int level,
- Text nullSequence, boolean escaped, byte escapeChar, boolean[] needsEscape) throws IOException {
-
+ Text nullSequence, boolean escaped, byte escapeChar, boolean[] needsEscape)
+ throws IOException {
+
if (obj == null) {
out.write(nullSequence.getBytes(), 0, nullSequence.getLength());
return;
}
-
+
switch (objInspector.getCategory()) {
- case PRIMITIVE: {
- LazyUtils.writePrimitiveUTF8(out, obj, (PrimitiveObjectInspector)objInspector, escaped, escapeChar, needsEscape);
- return;
- }
- case LIST: {
- char separator = (char)separators[level];
- ListObjectInspector loi = (ListObjectInspector)objInspector;
- List> list = loi.getList(obj);
- ObjectInspector eoi = loi.getListElementObjectInspector();
- if (list == null) {
- out.write(nullSequence.getBytes(), 0, nullSequence.getLength());
- } else {
- for (int i=0; i0) {
- out.write(separator);
- }
- serialize(out, list.get(i), eoi, separators, level+1,
- nullSequence, escaped, escapeChar, needsEscape);
+ case PRIMITIVE: {
+ LazyUtils.writePrimitiveUTF8(out, obj,
+ (PrimitiveObjectInspector) objInspector, escaped, escapeChar,
+ needsEscape);
+ return;
+ }
+ case LIST: {
+ char separator = (char) separators[level];
+ ListObjectInspector loi = (ListObjectInspector) objInspector;
+ List> list = loi.getList(obj);
+ ObjectInspector eoi = loi.getListElementObjectInspector();
+ if (list == null) {
+ out.write(nullSequence.getBytes(), 0, nullSequence.getLength());
+ } else {
+ for (int i = 0; i < list.size(); i++) {
+ if (i > 0) {
+ out.write(separator);
}
+ serialize(out, list.get(i), eoi, separators, level + 1, nullSequence,
+ escaped, escapeChar, needsEscape);
}
- return;
}
- case MAP: {
- char separator = (char)separators[level];
- char keyValueSeparator = (char)separators[level+1];
- MapObjectInspector moi = (MapObjectInspector)objInspector;
- ObjectInspector koi = moi.getMapKeyObjectInspector();
- ObjectInspector voi = moi.getMapValueObjectInspector();
-
- Map, ?> map = moi.getMap(obj);
- if (map == null) {
- out.write(nullSequence.getBytes(), 0, nullSequence.getLength());
- } else {
- boolean first = true;
- for (Map.Entry, ?> entry: map.entrySet()) {
- if (first) {
- first = false;
- } else {
- out.write(separator);
- }
- serialize(out, entry.getKey(), koi, separators, level+2,
- nullSequence, escaped, escapeChar, needsEscape);
- out.write(keyValueSeparator);
- serialize(out, entry.getValue(), voi, separators, level+2,
- nullSequence, escaped, escapeChar, needsEscape);
+ return;
+ }
+ case MAP: {
+ char separator = (char) separators[level];
+ char keyValueSeparator = (char) separators[level + 1];
+ MapObjectInspector moi = (MapObjectInspector) objInspector;
+ ObjectInspector koi = moi.getMapKeyObjectInspector();
+ ObjectInspector voi = moi.getMapValueObjectInspector();
+
+ Map, ?> map = moi.getMap(obj);
+ if (map == null) {
+ out.write(nullSequence.getBytes(), 0, nullSequence.getLength());
+ } else {
+ boolean first = true;
+ for (Map.Entry, ?> entry : map.entrySet()) {
+ if (first) {
+ first = false;
+ } else {
+ out.write(separator);
}
+ serialize(out, entry.getKey(), koi, separators, level + 2,
+ nullSequence, escaped, escapeChar, needsEscape);
+ out.write(keyValueSeparator);
+ serialize(out, entry.getValue(), voi, separators, level + 2,
+ nullSequence, escaped, escapeChar, needsEscape);
}
- return;
}
- case STRUCT: {
- char separator = (char)separators[level];
- StructObjectInspector soi = (StructObjectInspector)objInspector;
- List extends StructField> fields = soi.getAllStructFieldRefs();
- List list = soi.getStructFieldsDataAsList(obj);
- if (list == null) {
- out.write(nullSequence.getBytes(), 0, nullSequence.getLength());
- } else {
- for (int i=0; i0) {
- out.write(separator);
- }
- serialize(out, list.get(i),
- fields.get(i).getFieldObjectInspector(), separators, level+1,
- nullSequence, escaped, escapeChar, needsEscape);
+ return;
+ }
+ case STRUCT: {
+ char separator = (char) separators[level];
+ StructObjectInspector soi = (StructObjectInspector) objInspector;
+ List extends StructField> fields = soi.getAllStructFieldRefs();
+ List list = soi.getStructFieldsDataAsList(obj);
+ if (list == null) {
+ out.write(nullSequence.getBytes(), 0, nullSequence.getLength());
+ } else {
+ for (int i = 0; i < list.size(); i++) {
+ if (i > 0) {
+ out.write(separator);
}
+ serialize(out, list.get(i), fields.get(i).getFieldObjectInspector(),
+ separators, level + 1, nullSequence, escaped, escapeChar,
+ needsEscape);
}
- return;
}
+ return;
}
-
+ }
+
throw new RuntimeException("Unknown category type: "
+ objInspector.getCategory());
}
@@ -526,7 +554,8 @@
}
/**
- * @param useJSONSerialize the useJSONSerialize to set
+ * @param useJSONSerialize
+ * the useJSONSerialize to set
*/
public void setUseJSONSerialize(boolean useJSONSerialize) {
this.useJSONSerialize = useJSONSerialize;
Index: serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazyObjectInspectorFactory.java
===================================================================
--- serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazyObjectInspectorFactory.java (revision 901519)
+++ serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazyObjectInspectorFactory.java (working copy)
@@ -21,37 +21,30 @@
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
-import java.util.Map;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.UnionStructObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableBooleanObjectInspector;
import org.apache.hadoop.io.Text;
-
/**
- * ObjectInspectorFactory is the primary way to create new ObjectInspector instances.
+ * ObjectInspectorFactory is the primary way to create new ObjectInspector
+ * instances.
*
- * SerDe classes should call the static functions in this library to create an ObjectInspector
- * to return to the caller of SerDe2.getObjectInspector().
+ * SerDe classes should call the static functions in this library to create an
+ * ObjectInspector to return to the caller of SerDe2.getObjectInspector().
*
- * The reason of having caches here is that ObjectInspectors do not have an internal
- * state - so ObjectInspectors with the same construction parameters should
- * result in exactly the same ObjectInspector.
+ * The reason of having caches here is that ObjectInspectors do not have an
+ * internal state - so ObjectInspectors with the same construction parameters
+ * should result in exactly the same ObjectInspector.
*/
public class LazyObjectInspectorFactory {
+ static HashMap, LazySimpleStructObjectInspector> cachedLazySimpleStructObjectInspector = new HashMap, LazySimpleStructObjectInspector>();
-
- static HashMap, LazySimpleStructObjectInspector> cachedLazySimpleStructObjectInspector =
- new HashMap, LazySimpleStructObjectInspector>();
- public static LazySimpleStructObjectInspector getLazySimpleStructObjectInspector(List structFieldNames,
- List structFieldObjectInspectors, byte separator, Text nullSequence,
- boolean lastColumnTakesRest, boolean escaped, byte escapeChar) {
+ public static LazySimpleStructObjectInspector getLazySimpleStructObjectInspector(
+ List structFieldNames,
+ List structFieldObjectInspectors, byte separator,
+ Text nullSequence, boolean lastColumnTakesRest, boolean escaped,
+ byte escapeChar) {
ArrayList signature = new ArrayList();
signature.add(structFieldNames);
signature.add(structFieldObjectInspectors);
@@ -60,40 +53,44 @@
signature.add(Boolean.valueOf(lastColumnTakesRest));
signature.add(Boolean.valueOf(escaped));
signature.add(Byte.valueOf(escapeChar));
- LazySimpleStructObjectInspector result = cachedLazySimpleStructObjectInspector.get(signature);
+ LazySimpleStructObjectInspector result = cachedLazySimpleStructObjectInspector
+ .get(signature);
if (result == null) {
- result = new LazySimpleStructObjectInspector(structFieldNames, structFieldObjectInspectors,
- separator, nullSequence, lastColumnTakesRest, escaped, escapeChar);
+ result = new LazySimpleStructObjectInspector(structFieldNames,
+ structFieldObjectInspectors, separator, nullSequence,
+ lastColumnTakesRest, escaped, escapeChar);
cachedLazySimpleStructObjectInspector.put(signature, result);
}
return result;
}
- static HashMap, LazyListObjectInspector> cachedLazySimpleListObjectInspector =
- new HashMap, LazyListObjectInspector>();
- public static LazyListObjectInspector getLazySimpleListObjectInspector(
- ObjectInspector listElementObjectInspector, byte separator, Text nullSequence,
- boolean escaped, byte escapeChar) {
+ static HashMap, LazyListObjectInspector> cachedLazySimpleListObjectInspector = new HashMap, LazyListObjectInspector>();
+
+ public static LazyListObjectInspector getLazySimpleListObjectInspector(
+ ObjectInspector listElementObjectInspector, byte separator,
+ Text nullSequence, boolean escaped, byte escapeChar) {
ArrayList signature = new ArrayList();
signature.add(listElementObjectInspector);
signature.add(Byte.valueOf(separator));
signature.add(nullSequence.toString());
signature.add(Boolean.valueOf(escaped));
signature.add(Byte.valueOf(escapeChar));
- LazyListObjectInspector result = cachedLazySimpleListObjectInspector.get(signature);
+ LazyListObjectInspector result = cachedLazySimpleListObjectInspector
+ .get(signature);
if (result == null) {
- result = new LazyListObjectInspector(listElementObjectInspector,
+ result = new LazyListObjectInspector(listElementObjectInspector,
separator, nullSequence, escaped, escapeChar);
cachedLazySimpleListObjectInspector.put(signature, result);
}
return result;
}
-
- static HashMap, LazyMapObjectInspector> cachedLazySimpleMapObjectInspector =
- new HashMap, LazyMapObjectInspector>();
- public static LazyMapObjectInspector getLazySimpleMapObjectInspector(
- ObjectInspector mapKeyObjectInspector, ObjectInspector mapValueObjectInspector,
- byte itemSeparator, byte keyValueSeparator, Text nullSequence, boolean escaped,
+
+ static HashMap, LazyMapObjectInspector> cachedLazySimpleMapObjectInspector = new HashMap, LazyMapObjectInspector>();
+
+ public static LazyMapObjectInspector getLazySimpleMapObjectInspector(
+ ObjectInspector mapKeyObjectInspector,
+ ObjectInspector mapValueObjectInspector, byte itemSeparator,
+ byte keyValueSeparator, Text nullSequence, boolean escaped,
byte escapeChar) {
ArrayList signature = new ArrayList();
signature.add(mapKeyObjectInspector);
@@ -103,14 +100,15 @@
signature.add(nullSequence.toString());
signature.add(Boolean.valueOf(escaped));
signature.add(Byte.valueOf(escapeChar));
- LazyMapObjectInspector result = cachedLazySimpleMapObjectInspector.get(signature);
+ LazyMapObjectInspector result = cachedLazySimpleMapObjectInspector
+ .get(signature);
if (result == null) {
result = new LazyMapObjectInspector(mapKeyObjectInspector,
- mapValueObjectInspector, itemSeparator, keyValueSeparator, nullSequence,
- escaped, escapeChar);
+ mapValueObjectInspector, itemSeparator, keyValueSeparator,
+ nullSequence, escaped, escapeChar);
cachedLazySimpleMapObjectInspector.put(signature, result);
}
return result;
}
-
+
}
Index: serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyVoidObjectInspector.java
===================================================================
--- serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyVoidObjectInspector.java (revision 901519)
+++ serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyVoidObjectInspector.java (working copy)
@@ -21,17 +21,17 @@
import org.apache.hadoop.hive.serde2.objectinspector.primitive.VoidObjectInspector;
import org.apache.hadoop.io.NullWritable;
-
/**
* A WritableVoidObjectInspector inspects a NullWritable Object.
*/
-public class LazyVoidObjectInspector extends AbstractPrimitiveLazyObjectInspector
-implements VoidObjectInspector{
+public class LazyVoidObjectInspector extends
+ AbstractPrimitiveLazyObjectInspector implements
+ VoidObjectInspector {
LazyVoidObjectInspector() {
super(PrimitiveObjectInspectorUtils.voidTypeEntry);
}
-
+
@Override
public Object copyObject(Object o) {
return o;
Index: serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyByteObjectInspector.java
===================================================================
--- serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyByteObjectInspector.java (revision 901519)
+++ serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyByteObjectInspector.java (working copy)
@@ -21,19 +21,18 @@
import org.apache.hadoop.hive.serde2.lazy.LazyByte;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.ByteObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils;
-import org.apache.hadoop.io.BooleanWritable;
-
/**
* A WritableByteObjectInspector inspects a ByteWritable Object.
*/
-public class LazyByteObjectInspector extends AbstractPrimitiveLazyObjectInspector
-implements ByteObjectInspector{
+public class LazyByteObjectInspector extends
+ AbstractPrimitiveLazyObjectInspector implements
+ ByteObjectInspector {
LazyByteObjectInspector() {
super(PrimitiveObjectInspectorUtils.byteTypeEntry);
}
-
+
@Override
public byte get(Object o) {
return getPrimitiveWritableObject(o).get();
@@ -41,7 +40,7 @@
@Override
public Object copyObject(Object o) {
- return o == null ? null : new LazyByte((LazyByte)o);
+ return o == null ? null : new LazyByte((LazyByte) o);
}
@Override
Index: serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyFloatObjectInspector.java
===================================================================
--- serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyFloatObjectInspector.java (revision 901519)
+++ serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyFloatObjectInspector.java (working copy)
@@ -25,13 +25,14 @@
/**
* A FloatObjectInspector inspects a FloatWritable Object.
*/
-public class LazyFloatObjectInspector extends AbstractPrimitiveLazyObjectInspector
-implements FloatObjectInspector{
+public class LazyFloatObjectInspector extends
+ AbstractPrimitiveLazyObjectInspector implements
+ FloatObjectInspector {
LazyFloatObjectInspector() {
super(PrimitiveObjectInspectorUtils.floatTypeEntry);
}
-
+
@Override
public float get(Object o) {
return getPrimitiveWritableObject(o).get();
@@ -39,7 +40,7 @@
@Override
public Object copyObject(Object o) {
- return o == null ? null : new LazyFloat((LazyFloat)o);
+ return o == null ? null : new LazyFloat((LazyFloat) o);
}
@Override
Index: serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyStringObjectInspector.java
===================================================================
--- serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyStringObjectInspector.java (revision 901519)
+++ serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyStringObjectInspector.java (working copy)
@@ -22,16 +22,15 @@
import org.apache.hadoop.hive.serde2.objectinspector.primitive.StringObjectInspector;
import org.apache.hadoop.io.Text;
-
/**
* A WritableStringObjectInspector inspects a Text Object.
*/
-public class LazyStringObjectInspector extends AbstractPrimitiveLazyObjectInspector
-implements StringObjectInspector{
+public class LazyStringObjectInspector extends
+ AbstractPrimitiveLazyObjectInspector implements StringObjectInspector {
boolean escaped;
byte escapeChar;
-
+
LazyStringObjectInspector(boolean escaped, byte escapeChar) {
super(PrimitiveObjectInspectorUtils.stringTypeEntry);
this.escaped = escaped;
@@ -40,25 +39,25 @@
@Override
public Object copyObject(Object o) {
- return o == null ? null : new LazyString((LazyString)o);
+ return o == null ? null : new LazyString((LazyString) o);
}
@Override
public Text getPrimitiveWritableObject(Object o) {
- return o == null ? null : ((LazyString)o).getWritableObject();
+ return o == null ? null : ((LazyString) o).getWritableObject();
}
-
+
@Override
public String getPrimitiveJavaObject(Object o) {
- return o == null ? null : ((LazyString)o).getWritableObject().toString();
+ return o == null ? null : ((LazyString) o).getWritableObject().toString();
}
public boolean isEscaped() {
return escaped;
}
+
public byte getEscapeChar() {
return escapeChar;
}
-
-
+
}
Index: serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyDoubleObjectInspector.java
===================================================================
--- serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyDoubleObjectInspector.java (revision 901519)
+++ serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyDoubleObjectInspector.java (working copy)
@@ -22,17 +22,17 @@
import org.apache.hadoop.hive.serde2.objectinspector.primitive.DoubleObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils;
-
/**
* A WritableDoubleObjectInspector inspects a DoubleWritable Object.
*/
-public class LazyDoubleObjectInspector extends AbstractPrimitiveLazyObjectInspector
-implements DoubleObjectInspector{
+public class LazyDoubleObjectInspector extends
+ AbstractPrimitiveLazyObjectInspector implements
+ DoubleObjectInspector {
LazyDoubleObjectInspector() {
super(PrimitiveObjectInspectorUtils.doubleTypeEntry);
}
-
+
@Override
public double get(Object o) {
return getPrimitiveWritableObject(o).get();
@@ -40,7 +40,7 @@
@Override
public Object copyObject(Object o) {
- return o == null ? null : new LazyDouble((LazyDouble)o);
+ return o == null ? null : new LazyDouble((LazyDouble) o);
}
@Override
Index: serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyIntObjectInspector.java
===================================================================
--- serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyIntObjectInspector.java (revision 901519)
+++ serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyIntObjectInspector.java (working copy)
@@ -22,17 +22,17 @@
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils;
import org.apache.hadoop.io.IntWritable;
-
/**
* A WritableIntObjectInspector inspects a IntWritable Object.
*/
-public class LazyIntObjectInspector extends AbstractPrimitiveLazyObjectInspector
-implements IntObjectInspector{
+public class LazyIntObjectInspector extends
+ AbstractPrimitiveLazyObjectInspector implements
+ IntObjectInspector {
LazyIntObjectInspector() {
super(PrimitiveObjectInspectorUtils.intTypeEntry);
}
-
+
@Override
public int get(Object o) {
return getPrimitiveWritableObject(o).get();
@@ -40,7 +40,7 @@
@Override
public Object copyObject(Object o) {
- return o == null ? null : new LazyInteger((LazyInteger)o);
+ return o == null ? null : new LazyInteger((LazyInteger) o);
}
@Override
Index: serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/AbstractPrimitiveLazyObjectInspector.java
===================================================================
--- serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/AbstractPrimitiveLazyObjectInspector.java (revision 901519)
+++ serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/AbstractPrimitiveLazyObjectInspector.java (working copy)
@@ -23,22 +23,23 @@
import org.apache.hadoop.io.Writable;
/**
- * An AbstractPrimitiveLazyObjectInspector for a LazyPrimitive object.
+ * An AbstractPrimitiveLazyObjectInspector for a LazyPrimitive object.
*/
-public abstract class AbstractPrimitiveLazyObjectInspector extends AbstractPrimitiveObjectInspector {
+public abstract class AbstractPrimitiveLazyObjectInspector
+ extends AbstractPrimitiveObjectInspector {
protected AbstractPrimitiveLazyObjectInspector(PrimitiveTypeEntry typeEntry) {
- super(typeEntry);
+ super(typeEntry);
}
-
+
@Override
public T getPrimitiveWritableObject(Object o) {
- return o == null ? null : ((LazyPrimitive,T>)o).getWritableObject();
+ return o == null ? null : ((LazyPrimitive, T>) o).getWritableObject();
}
@Override
public boolean preferWritable() {
return true;
}
-
+
}
Index: serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyBooleanObjectInspector.java
===================================================================
--- serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyBooleanObjectInspector.java (revision 901519)
+++ serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyBooleanObjectInspector.java (working copy)
@@ -22,17 +22,17 @@
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils;
import org.apache.hadoop.io.BooleanWritable;
-
/**
* A WritableBooleanObjectInspector inspects a BooleanWritable Object.
*/
-public class LazyBooleanObjectInspector extends AbstractPrimitiveLazyObjectInspector
-implements BooleanObjectInspector{
+public class LazyBooleanObjectInspector extends
+ AbstractPrimitiveLazyObjectInspector implements
+ BooleanObjectInspector {
LazyBooleanObjectInspector() {
super(PrimitiveObjectInspectorUtils.booleanTypeEntry);
}
-
+
@Override
public boolean get(Object o) {
return getPrimitiveWritableObject(o).get();
@@ -40,7 +40,7 @@
@Override
public Object copyObject(Object o) {
- return o == null ? null : new LazyBoolean((LazyBoolean)o);
+ return o == null ? null : new LazyBoolean((LazyBoolean) o);
}
@Override
Index: serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyLongObjectInspector.java
===================================================================
--- serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyLongObjectInspector.java (revision 901519)
+++ serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyLongObjectInspector.java (working copy)
@@ -22,17 +22,17 @@
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils;
import org.apache.hadoop.io.LongWritable;
-
/**
* A WritableLongObjectInspector inspects a LongWritable Object.
*/
-public class LazyLongObjectInspector extends AbstractPrimitiveLazyObjectInspector
-implements LongObjectInspector{
+public class LazyLongObjectInspector extends
+ AbstractPrimitiveLazyObjectInspector implements
+ LongObjectInspector {
LazyLongObjectInspector() {
super(PrimitiveObjectInspectorUtils.longTypeEntry);
}
-
+
@Override
public long get(Object o) {
return getPrimitiveWritableObject(o).get();
@@ -40,7 +40,7 @@
@Override
public Object copyObject(Object o) {
- return o == null ? null : new LazyLong((LazyLong)o);
+ return o == null ? null : new LazyLong((LazyLong) o);
}
@Override
Index: serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyShortObjectInspector.java
===================================================================
--- serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyShortObjectInspector.java (revision 901519)
+++ serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyShortObjectInspector.java (working copy)
@@ -22,17 +22,17 @@
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.ShortObjectInspector;
-
/**
* A WritableShortObjectInspector inspects a ShortWritable Object.
*/
-public class LazyShortObjectInspector extends AbstractPrimitiveLazyObjectInspector
-implements ShortObjectInspector{
+public class LazyShortObjectInspector extends
+ AbstractPrimitiveLazyObjectInspector implements
+ ShortObjectInspector {
LazyShortObjectInspector() {
super(PrimitiveObjectInspectorUtils.shortTypeEntry);
}
-
+
@Override
public short get(Object o) {
return getPrimitiveWritableObject(o).get();
@@ -40,7 +40,7 @@
@Override
public Object copyObject(Object o) {
- return o == null ? null : new LazyShort((LazyShort)o);
+ return o == null ? null : new LazyShort((LazyShort) o);
}
@Override
Index: serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyPrimitiveObjectInspectorFactory.java
===================================================================
--- serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyPrimitiveObjectInspectorFactory.java (revision 901519)
+++ serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyPrimitiveObjectInspectorFactory.java (working copy)
@@ -20,37 +20,20 @@
import java.util.ArrayList;
import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.UnionStructObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.AbstractPrimitiveJavaObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableBooleanObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableByteObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableDoubleObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableFloatObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableIntObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableLongObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableShortObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableStringObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableVoidObjectInspector;
-import org.apache.hadoop.io.Text;
-
/**
- * LazyPrimitiveObjectInspectorFactory is the primary way to create new ObjectInspector instances.
+ * LazyPrimitiveObjectInspectorFactory is the primary way to create new
+ * ObjectInspector instances.
*
- * SerDe classes should call the static functions in this library to create an ObjectInspector
- * to return to the caller of SerDe2.getObjectInspector().
+ * SerDe classes should call the static functions in this library to create an
+ * ObjectInspector to return to the caller of SerDe2.getObjectInspector().
*
- * The reason of having caches here is that ObjectInspector is because ObjectInspectors do
- * not have an internal state - so ObjectInspectors with the same construction parameters should
- * result in exactly the same ObjectInspector.
+ * The reason of having caches here is that ObjectInspector is because
+ * ObjectInspectors do not have an internal state - so ObjectInspectors with the
+ * same construction parameters should result in exactly the same
+ * ObjectInspector.
*/
public class LazyPrimitiveObjectInspectorFactory {
@@ -62,38 +45,48 @@
public final static LazyFloatObjectInspector lazyFloatObjectInspector = new LazyFloatObjectInspector();
public final static LazyDoubleObjectInspector lazyDoubleObjectInspector = new LazyDoubleObjectInspector();
public final static LazyVoidObjectInspector lazyVoidObjectInspector = new LazyVoidObjectInspector();
-
- static HashMap, LazyStringObjectInspector> cachedLazyStringObjectInspector =
- new HashMap, LazyStringObjectInspector>();
- public static LazyStringObjectInspector getLazyStringObjectInspector(boolean escaped, byte escapeChar) {
+
+ static HashMap, LazyStringObjectInspector> cachedLazyStringObjectInspector = new HashMap, LazyStringObjectInspector>();
+
+ public static LazyStringObjectInspector getLazyStringObjectInspector(
+ boolean escaped, byte escapeChar) {
ArrayList signature = new ArrayList();
signature.add(Boolean.valueOf(escaped));
signature.add(Byte.valueOf(escapeChar));
- LazyStringObjectInspector result = cachedLazyStringObjectInspector.get(signature);
+ LazyStringObjectInspector result = cachedLazyStringObjectInspector
+ .get(signature);
if (result == null) {
result = new LazyStringObjectInspector(escaped, escapeChar);
cachedLazyStringObjectInspector.put(signature, result);
}
return result;
}
-
+
public static AbstractPrimitiveLazyObjectInspector> getLazyObjectInspector(
PrimitiveCategory primitiveCategory, boolean escaped, byte escapeChar) {
-
- switch(primitiveCategory) {
- case BOOLEAN: return lazyBooleanObjectInspector;
- case BYTE: return lazyByteObjectInspector;
- case SHORT: return lazyShortObjectInspector;
- case INT: return lazyIntObjectInspector;
- case LONG: return lazyLongObjectInspector;
- case FLOAT: return lazyFloatObjectInspector;
- case DOUBLE: return lazyDoubleObjectInspector;
- case STRING: return getLazyStringObjectInspector(escaped, escapeChar);
+
+ switch (primitiveCategory) {
+ case BOOLEAN:
+ return lazyBooleanObjectInspector;
+ case BYTE:
+ return lazyByteObjectInspector;
+ case SHORT:
+ return lazyShortObjectInspector;
+ case INT:
+ return lazyIntObjectInspector;
+ case LONG:
+ return lazyLongObjectInspector;
+ case FLOAT:
+ return lazyFloatObjectInspector;
+ case DOUBLE:
+ return lazyDoubleObjectInspector;
+ case STRING:
+ return getLazyStringObjectInspector(escaped, escapeChar);
case VOID:
- default:
- throw new RuntimeException("Internal error: Cannot find ObjectInspector "
- + " for " + primitiveCategory);
+ default:
+ throw new RuntimeException("Internal error: Cannot find ObjectInspector "
+ + " for " + primitiveCategory);
}
}
-
+
}
Index: serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazyListObjectInspector.java
===================================================================
--- serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazyListObjectInspector.java (revision 901519)
+++ serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazyListObjectInspector.java (working copy)
@@ -30,23 +30,25 @@
/**
* LazyListObjectInspector works on array data that is stored in LazyArray.
*
- * Always use the ObjectInspectorFactory to create new ObjectInspector objects, instead
- * of directly creating an instance of this class.
+ * Always use the ObjectInspectorFactory to create new ObjectInspector objects,
+ * instead of directly creating an instance of this class.
*/
public class LazyListObjectInspector implements ListObjectInspector {
- public static final Log LOG = LogFactory.getLog(LazyListObjectInspector.class.getName());
-
+ public static final Log LOG = LogFactory.getLog(LazyListObjectInspector.class
+ .getName());
+
ObjectInspector listElementObjectInspector;
-
+
byte separator;
Text nullSequence;
boolean escaped;
byte escapeChar;
-
- /** Call ObjectInspectorFactory.getLazySimpleListObjectInspector instead.
+
+ /**
+ * Call ObjectInspectorFactory.getLazySimpleListObjectInspector instead.
*/
- protected LazyListObjectInspector(ObjectInspector listElementObjectInspector,
+ protected LazyListObjectInspector(ObjectInspector listElementObjectInspector,
byte separator, Text nullSequence, boolean escaped, byte escapeChar) {
this.listElementObjectInspector = listElementObjectInspector;
this.separator = separator;
@@ -65,7 +67,7 @@
public ObjectInspector getListElementObjectInspector() {
return listElementObjectInspector;
}
-
+
// with data
@Override
public Object getListElement(Object data, int index) {
@@ -75,7 +77,7 @@
LazyArray array = (LazyArray) data;
return array.getListElementObject(index);
}
-
+
@Override
public int getListLength(Object data) {
if (data == null) {
@@ -84,7 +86,7 @@
LazyArray array = (LazyArray) data;
return array.getListLength();
}
-
+
@Override
public List> getList(Object data) {
if (data == null) {
@@ -96,28 +98,29 @@
@Override
public String getTypeName() {
- return org.apache.hadoop.hive.serde.Constants.LIST_TYPE_NAME
- + "<" + listElementObjectInspector.getTypeName() + ">";
+ return org.apache.hadoop.hive.serde.Constants.LIST_TYPE_NAME + "<"
+ + listElementObjectInspector.getTypeName() + ">";
}
/**
- * Returns the separator for delimiting items in this array.
- * Called by LazyArray.init(...).
+ * Returns the separator for delimiting items in this array. Called by
+ * LazyArray.init(...).
*/
public byte getSeparator() {
return separator;
}
/**
- * Returns the NullSequence for this array.
- * Called by LazyArray.init(...).
+ * Returns the NullSequence for this array. Called by LazyArray.init(...).
*/
public Text getNullSequence() {
return nullSequence;
}
+
public boolean isEscaped() {
return escaped;
}
+
public byte getEscapeChar() {
return escapeChar;
}
Index: serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazyMapObjectInspector.java
===================================================================
--- serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazyMapObjectInspector.java (revision 901519)
+++ serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazyMapObjectInspector.java (working copy)
@@ -30,28 +30,30 @@
/**
* LazyMapObjectInspector works on struct data that is stored in LazyStruct.
*
- * Always use the ObjectInspectorFactory to create new ObjectInspector objects, instead
- * of directly creating an instance of this class.
+ * Always use the ObjectInspectorFactory to create new ObjectInspector objects,
+ * instead of directly creating an instance of this class.
*/
public class LazyMapObjectInspector implements MapObjectInspector {
- public static final Log LOG = LogFactory.getLog(LazyMapObjectInspector.class.getName());
-
+ public static final Log LOG = LogFactory.getLog(LazyMapObjectInspector.class
+ .getName());
+
ObjectInspector mapKeyObjectInspector;
ObjectInspector mapValueObjectInspector;
-
+
byte itemSeparator;
- byte keyValueSeparator;
+ byte keyValueSeparator;
Text nullSequence;
boolean escaped;
byte escapeChar;
-
- /** Call ObjectInspectorFactory.getStandardListObjectInspector instead.
+
+ /**
+ * Call ObjectInspectorFactory.getStandardListObjectInspector instead.
*/
protected LazyMapObjectInspector(ObjectInspector mapKeyObjectInspector,
- ObjectInspector mapValueObjectInspector,
- byte itemSeparator, byte keyValueSeparator, Text nullSequence,
- boolean escaped, byte escapeChar) {
+ ObjectInspector mapValueObjectInspector, byte itemSeparator,
+ byte keyValueSeparator, Text nullSequence, boolean escaped,
+ byte escapeChar) {
this.mapKeyObjectInspector = mapKeyObjectInspector;
this.mapValueObjectInspector = mapValueObjectInspector;
@@ -69,8 +71,8 @@
@Override
public String getTypeName() {
- return org.apache.hadoop.hive.serde.Constants.MAP_TYPE_NAME
- + "<" + mapKeyObjectInspector.getTypeName() + ","
+ return org.apache.hadoop.hive.serde.Constants.MAP_TYPE_NAME + "<"
+ + mapKeyObjectInspector.getTypeName() + ","
+ mapValueObjectInspector.getTypeName() + ">";
}
@@ -89,7 +91,7 @@
if (data == null) {
return null;
}
- return ((LazyMap)data).getMapValueElement(key);
+ return ((LazyMap) data).getMapValueElement(key);
}
@Override
@@ -97,7 +99,7 @@
if (data == null) {
return null;
}
- return ((LazyMap)data).getMap();
+ return ((LazyMap) data).getMap();
}
@Override
@@ -105,22 +107,26 @@
if (data == null) {
return -1;
}
- return ((LazyMap)data).getMapSize();
+ return ((LazyMap) data).getMapSize();
}
-
+
// Called by LazyMap
public byte getItemSeparator() {
return itemSeparator;
}
+
public byte getKeyValueSeparator() {
return keyValueSeparator;
}
+
public Text getNullSequence() {
return nullSequence;
}
+
public boolean isEscaped() {
return escaped;
}
+
public byte getEscapeChar() {
return escapeChar;
}
Index: serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazySimpleStructObjectInspector.java
===================================================================
--- serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazySimpleStructObjectInspector.java (revision 901519)
+++ serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazySimpleStructObjectInspector.java (working copy)
@@ -31,24 +31,27 @@
import org.apache.hadoop.io.Text;
/**
- * LazySimpleStructObjectInspector works on struct data that is stored in LazyStruct.
+ * LazySimpleStructObjectInspector works on struct data that is stored in
+ * LazyStruct.
*
- * The names of the struct fields and the internal structure of the struct fields
- * are specified in the ctor of the LazySimpleStructObjectInspector.
+ * The names of the struct fields and the internal structure of the struct
+ * fields are specified in the ctor of the LazySimpleStructObjectInspector.
*
- * Always use the ObjectInspectorFactory to create new ObjectInspector objects, instead
- * of directly creating an instance of this class.
+ * Always use the ObjectInspectorFactory to create new ObjectInspector objects,
+ * instead of directly creating an instance of this class.
*/
public class LazySimpleStructObjectInspector extends StructObjectInspector {
- public static final Log LOG = LogFactory.getLog(LazySimpleStructObjectInspector.class.getName());
-
+ public static final Log LOG = LogFactory
+ .getLog(LazySimpleStructObjectInspector.class.getName());
+
protected static class MyField implements StructField {
protected int fieldID;
protected String fieldName;
protected ObjectInspector fieldObjectInspector;
-
- public MyField(int fieldID, String fieldName, ObjectInspector fieldObjectInspector) {
+
+ public MyField(int fieldID, String fieldName,
+ ObjectInspector fieldObjectInspector) {
this.fieldID = fieldID;
this.fieldName = fieldName.toLowerCase();
this.fieldObjectInspector = fieldObjectInspector;
@@ -57,70 +60,81 @@
public int getFieldID() {
return fieldID;
}
+
public String getFieldName() {
return fieldName;
}
+
public ObjectInspector getFieldObjectInspector() {
return fieldObjectInspector;
}
-
+
+ @Override
public String toString() {
return "" + fieldID + ":" + fieldName;
}
}
-
+
protected List fields;
-
+
@Override
public String getTypeName() {
return ObjectInspectorUtils.getStandardStructTypeName(this);
}
-
-
+
byte separator;
- Text nullSequence;
+ Text nullSequence;
boolean lastColumnTakesRest;
boolean escaped;
byte escapeChar;
-
- /** Call ObjectInspectorFactory.getLazySimpleStructObjectInspector instead.
+
+ /**
+ * Call ObjectInspectorFactory.getLazySimpleStructObjectInspector instead.
*/
- protected LazySimpleStructObjectInspector(List structFieldNames, List structFieldObjectInspectors,
- byte separator, Text nullSequence, boolean lastColumnTakesRest,
- boolean escaped, byte escapeChar) {
- init(structFieldNames, structFieldObjectInspectors, separator, nullSequence, lastColumnTakesRest,
- escaped, escapeChar);
+ protected LazySimpleStructObjectInspector(List structFieldNames,
+ List structFieldObjectInspectors, byte separator,
+ Text nullSequence, boolean lastColumnTakesRest, boolean escaped,
+ byte escapeChar) {
+ init(structFieldNames, structFieldObjectInspectors, separator,
+ nullSequence, lastColumnTakesRest, escaped, escapeChar);
}
- protected void init(List structFieldNames, List structFieldObjectInspectors,
- byte separator, Text nullSequence, boolean lastColumnTakesRest, boolean escaped, byte escapeChar) {
- assert(structFieldNames.size() == structFieldObjectInspectors.size());
-
+
+ protected void init(List structFieldNames,
+ List structFieldObjectInspectors, byte separator,
+ Text nullSequence, boolean lastColumnTakesRest, boolean escaped,
+ byte escapeChar) {
+ assert (structFieldNames.size() == structFieldObjectInspectors.size());
+
this.separator = separator;
this.nullSequence = nullSequence;
this.lastColumnTakesRest = lastColumnTakesRest;
this.escaped = escaped;
this.escapeChar = escapeChar;
-
- fields = new ArrayList(structFieldNames.size());
- for(int i=0; i(structFieldNames.size());
+ for (int i = 0; i < structFieldNames.size(); i++) {
+ fields.add(new MyField(i, structFieldNames.get(i),
+ structFieldObjectInspectors.get(i)));
}
}
-
- protected LazySimpleStructObjectInspector(List fields, byte separator, Text nullSequence) {
+
+ protected LazySimpleStructObjectInspector(List fields,
+ byte separator, Text nullSequence) {
init(fields, separator, nullSequence);
}
- protected void init(List fields, byte separator, Text nullSequence) {
+
+ protected void init(List fields, byte separator,
+ Text nullSequence) {
this.separator = separator;
this.nullSequence = nullSequence;
-
- this.fields = new ArrayList(fields.size());
- for(int i=0; i(fields.size());
+ for (int i = 0; i < fields.size(); i++) {
+ this.fields.add(new MyField(i, fields.get(i).getFieldName(), fields
+ .get(i).getFieldObjectInspector()));
}
}
-
@Override
public final Category getCategory() {
return Category.STRUCT;
@@ -131,6 +145,7 @@
public StructField getStructFieldRef(String fieldName) {
return ObjectInspectorUtils.getStandardStructFieldRef(fieldName, fields);
}
+
@Override
public List extends StructField> getAllStructFieldRefs() {
return fields;
@@ -142,12 +157,12 @@
if (data == null) {
return null;
}
- LazyStruct struct = (LazyStruct)data;
+ LazyStruct struct = (LazyStruct) data;
MyField f = (MyField) fieldRef;
-
+
int fieldID = f.getFieldID();
- assert(fieldID >= 0 && fieldID < fields.size());
-
+ assert (fieldID >= 0 && fieldID < fields.size());
+
return struct.getField(fieldID);
}
@@ -156,7 +171,7 @@
if (data == null) {
return null;
}
- LazyStruct struct = (LazyStruct)data;
+ LazyStruct struct = (LazyStruct) data;
return struct.getFieldsAsList();
}
@@ -164,17 +179,21 @@
public byte getSeparator() {
return separator;
}
+
public Text getNullSequence() {
return nullSequence;
}
+
public boolean getLastColumnTakesRest() {
return lastColumnTakesRest;
}
+
public boolean isEscaped() {
return escaped;
}
+
public byte getEscapeChar() {
return escapeChar;
}
-
+
}
Index: serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyMap.java
===================================================================
--- serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyMap.java (revision 901519)
+++ serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyMap.java (working copy)
@@ -18,7 +18,6 @@
package org.apache.hadoop.hive.serde2.lazy;
import java.util.Arrays;
-import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.Map;
@@ -28,59 +27,55 @@
import org.apache.hadoop.io.Text;
/**
- * LazyMap stores a map of Primitive LazyObjects to LazyObjects.
- * Note that the keys of the map cannot contain null.
+ * LazyMap stores a map of Primitive LazyObjects to LazyObjects. Note that the
+ * keys of the map cannot contain null.
*
- * LazyMap does not deal with the case of a NULL map. That is handled
- * by the parent LazyObject.
+ * LazyMap does not deal with the case of a NULL map. That is handled by the
+ * parent LazyObject.
*/
public class LazyMap extends LazyNonPrimitive {
-
+
/**
* Whether the data is already parsed or not.
*/
boolean parsed = false;
-
+
/**
- * The size of the map.
- * Only valid when the data is parsed.
- * -1 when the map is NULL.
+ * The size of the map. Only valid when the data is parsed. -1 when the map is
+ * NULL.
*/
int mapSize = 0;
-
+
/**
- * The beginning position of key[i].
- * Only valid when the data is parsed.
- * Note that keyStart[mapSize] = begin + length + 1;
- * that makes sure we can use the same formula to compute the
- * length of each value in the map.
+ * The beginning position of key[i]. Only valid when the data is parsed. Note
+ * that keyStart[mapSize] = begin + length + 1; that makes sure we can use the
+ * same formula to compute the length of each value in the map.
*/
int[] keyStart;
-
+
/**
- * The end position of key[i] (the position of the key-value separator).
- * Only valid when the data is parsed.
- */
+ * The end position of key[i] (the position of the key-value separator). Only
+ * valid when the data is parsed.
+ */
int[] keyEnd;
/**
* The keys are stored in an array of LazyPrimitives.
*/
- LazyPrimitive,?>[] keyObjects;
+ LazyPrimitive, ?>[] keyObjects;
/**
- * Whether init() is called on keyObjects[i].
+ * Whether init() is called on keyObjects[i].
*/
boolean[] keyInited;
/**
- * The values are stored in an array of LazyObjects.
- * value[index] will start from KeyEnd[index] + 1,
- * and ends before KeyStart[index+1] - 1.
+ * The values are stored in an array of LazyObjects. value[index] will start
+ * from KeyEnd[index] + 1, and ends before KeyStart[index+1] - 1.
*/
LazyObject[] valueObjects;
/**
* Whether init() is called on valueObjects[i]
*/
boolean[] valueInited;
-
+
/**
* Construct a LazyMap object with the ObjectInspector.
*/
@@ -90,6 +85,7 @@
/**
* Set the row data for this LazyArray.
+ *
* @see LazyObject#init(ByteArrayRef, int, int)
*/
@Override
@@ -97,27 +93,27 @@
super.init(bytes, start, length);
parsed = false;
}
-
+
/**
- * Enlarge the size of arrays storing information for the elements inside
- * the array.
+ * Enlarge the size of arrays storing information for the elements inside the
+ * array.
*/
protected void enlargeArrays() {
if (keyStart == null) {
int initialSize = 2;
keyStart = new int[initialSize];
keyEnd = new int[initialSize];
- keyObjects = new LazyPrimitive,?>[initialSize];
+ keyObjects = new LazyPrimitive, ?>[initialSize];
valueObjects = new LazyObject[initialSize];
keyInited = new boolean[initialSize];
valueInited = new boolean[initialSize];
} else {
- keyStart = Arrays.copyOf(keyStart, keyStart.length*2);
- keyEnd = Arrays.copyOf(keyEnd, keyEnd.length*2);
- keyObjects = Arrays.copyOf(keyObjects, keyObjects.length*2);
- valueObjects = Arrays.copyOf(valueObjects, valueObjects.length*2);
- keyInited = Arrays.copyOf(keyInited, keyInited.length*2);
- valueInited = Arrays.copyOf(valueInited, valueInited.length*2);
+ keyStart = Arrays.copyOf(keyStart, keyStart.length * 2);
+ keyEnd = Arrays.copyOf(keyEnd, keyEnd.length * 2);
+ keyObjects = Arrays.copyOf(keyObjects, keyObjects.length * 2);
+ valueObjects = Arrays.copyOf(valueObjects, valueObjects.length * 2);
+ keyInited = Arrays.copyOf(keyInited, keyInited.length * 2);
+ valueInited = Arrays.copyOf(valueInited, valueInited.length * 2);
}
}
@@ -126,44 +122,44 @@
*/
private void parse() {
parsed = true;
-
+
byte itemSeparator = oi.getItemSeparator();
- byte keyValueSeparator = oi.getKeyValueSeparator();
+ byte keyValueSeparator = oi.getKeyValueSeparator();
boolean isEscaped = oi.isEscaped();
byte escapeChar = oi.getEscapeChar();
-
+
// empty array?
if (length == 0) {
mapSize = 0;
return;
}
-
+
mapSize = 0;
int arrayByteEnd = start + length;
int elementByteBegin = start;
int keyValueSeparatorPosition = -1;
int elementByteEnd = start;
byte[] bytes = this.bytes.getData();
-
+
// Go through all bytes in the byte[]
while (elementByteEnd <= arrayByteEnd) {
// End of entry reached?
- if (elementByteEnd == arrayByteEnd
+ if (elementByteEnd == arrayByteEnd
|| bytes[elementByteEnd] == itemSeparator) {
// Array full?
if (keyStart == null || mapSize + 1 == keyStart.length) {
enlargeArrays();
}
keyStart[mapSize] = elementByteBegin;
- // If no keyValueSeparator is seen, all bytes belong to key, and
+ // If no keyValueSeparator is seen, all bytes belong to key, and
// value will be NULL.
- keyEnd[mapSize] = (keyValueSeparatorPosition == -1
- ? elementByteEnd: keyValueSeparatorPosition);
+ keyEnd[mapSize] = (keyValueSeparatorPosition == -1 ? elementByteEnd
+ : keyValueSeparatorPosition);
// reset keyValueSeparatorPosition
keyValueSeparatorPosition = -1;
mapSize++;
elementByteBegin = elementByteEnd + 1;
- elementByteEnd ++;
+ elementByteEnd++;
} else {
// Is this the first keyValueSeparator in this entry?
if (keyValueSeparatorPosition == -1
@@ -171,15 +167,15 @@
keyValueSeparatorPosition = elementByteEnd;
}
if (isEscaped && bytes[elementByteEnd] == escapeChar
- && elementByteEnd+1 < arrayByteEnd) {
+ && elementByteEnd + 1 < arrayByteEnd) {
// ignore the char after escape_char
elementByteEnd += 2;
} else {
- elementByteEnd ++;
+ elementByteEnd++;
}
}
}
-
+
// This makes sure we can use the same formula to compute the
// length of each value in the map.
keyStart[mapSize] = arrayByteEnd + 1;
@@ -189,19 +185,20 @@
Arrays.fill(valueInited, 0, mapSize, false);
}
}
-
+
/**
* Get the value in the map for the key.
*
- * If there are multiple matches (which is possible in the serialized
- * format), only the first one is returned.
+ * If there are multiple matches (which is possible in the serialized format),
+ * only the first one is returned.
*
- * The most efficient way to get the value for the key is to serialize the
- * key and then try to find it in the array. We do linear search because in
- * most cases, user only wants to get one or two values out of the map, and
- * the cost of building up a HashMap is substantially higher.
+ * The most efficient way to get the value for the key is to serialize the key
+ * and then try to find it in the array. We do linear search because in most
+ * cases, user only wants to get one or two values out of the map, and the
+ * cost of building up a HashMap is substantially higher.
*
- * @param key The key object that we are looking for.
+ * @param key
+ * The key object that we are looking for.
* @return The corresponding value object, or NULL if not found
*/
public Object getMapValueElement(Object key) {
@@ -209,83 +206,94 @@
parse();
}
// search for the key
- for (int i=0; i lazyKeyI = uncheckedGetKey(i);
- if (lazyKeyI == null) continue;
- // getWritableObject() will convert LazyPrimitive to actual primitive writable objects.
+ for (int i = 0; i < mapSize; i++) {
+ LazyPrimitive, ?> lazyKeyI = uncheckedGetKey(i);
+ if (lazyKeyI == null) {
+ continue;
+ }
+ // getWritableObject() will convert LazyPrimitive to actual primitive
+ // writable objects.
Object keyI = lazyKeyI.getWritableObject();
- if (keyI == null) continue;
+ if (keyI == null) {
+ continue;
+ }
if (keyI.equals(key)) {
// Got a match, return the value
LazyObject v = uncheckedGetValue(i);
return v == null ? v : v.getObject();
}
}
-
+
return null;
}
/**
* Get the value object with the index without checking parsed.
- * @param index The index into the array starting from 0
+ *
+ * @param index
+ * The index into the array starting from 0
*/
private LazyObject uncheckedGetValue(int index) {
Text nullSequence = oi.getNullSequence();
int valueIBegin = keyEnd[index] + 1;
- int valueILength = keyStart[index+1] - 1 - valueIBegin;
- if (valueILength < 0 ||
- ((valueILength == nullSequence.getLength())
- && 0 == LazyUtils.compare(bytes.getData(), valueIBegin, valueILength,
- nullSequence.getBytes(), 0, nullSequence.getLength()))) {
- return null;
+ int valueILength = keyStart[index + 1] - 1 - valueIBegin;
+ if (valueILength < 0
+ || ((valueILength == nullSequence.getLength()) && 0 == LazyUtils
+ .compare(bytes.getData(), valueIBegin, valueILength, nullSequence
+ .getBytes(), 0, nullSequence.getLength()))) {
+ return null;
}
if (!valueInited[index]) {
valueInited[index] = true;
if (valueObjects[index] == null) {
- valueObjects[index] = LazyFactory.createLazyObject(
- ((MapObjectInspector)oi).getMapValueObjectInspector());
+ valueObjects[index] = LazyFactory
+ .createLazyObject(((MapObjectInspector) oi)
+ .getMapValueObjectInspector());
}
valueObjects[index].init(bytes, valueIBegin, valueILength);
}
return valueObjects[index];
}
-
+
/**
* Get the key object with the index without checking parsed.
- * @param index The index into the array starting from 0
+ *
+ * @param index
+ * The index into the array starting from 0
*/
- private LazyPrimitive,?> uncheckedGetKey(int index) {
- Text nullSequence = oi.getNullSequence();
+ private LazyPrimitive, ?> uncheckedGetKey(int index) {
+ Text nullSequence = oi.getNullSequence();
int keyIBegin = keyStart[index];
int keyILength = keyEnd[index] - keyStart[index];
- if (keyILength < 0 ||
- ((keyILength == nullSequence.getLength())
- && 0 == LazyUtils.compare(bytes.getData(), keyIBegin, keyILength,
- nullSequence.getBytes(), 0, nullSequence.getLength()))) {
+ if (keyILength < 0
+ || ((keyILength == nullSequence.getLength()) && 0 == LazyUtils.compare(
+ bytes.getData(), keyIBegin, keyILength, nullSequence.getBytes(), 0,
+ nullSequence.getLength()))) {
return null;
}
if (!keyInited[index]) {
keyInited[index] = true;
if (keyObjects[index] == null) {
// Keys are always primitive
- keyObjects[index] = LazyFactory.createLazyPrimitiveClass(
- (PrimitiveObjectInspector)((MapObjectInspector)oi).getMapKeyObjectInspector());
+ keyObjects[index] = LazyFactory
+ .createLazyPrimitiveClass((PrimitiveObjectInspector) ((MapObjectInspector) oi)
+ .getMapKeyObjectInspector());
}
keyObjects[index].init(bytes, keyIBegin, keyILength);
}
return keyObjects[index];
}
-
+
/**
- * cachedMap is reused for different calls to getMap().
- * But each LazyMap has a separate cachedMap so we won't overwrite the
- * data by accident.
+ * cachedMap is reused for different calls to getMap(). But each LazyMap has a
+ * separate cachedMap so we won't overwrite the data by accident.
*/
LinkedHashMap cachedMap;
-
+
/**
- * Return the map object representing this LazyMap.
- * Note that the keyObjects will be Writable primitive objects.
+ * Return the map object representing this LazyMap. Note that the keyObjects
+ * will be Writable primitive objects.
+ *
* @return the map object
*/
public Map getMap() {
@@ -298,11 +306,13 @@
} else {
cachedMap.clear();
}
-
+
// go through each element of the map
for (int i = 0; i < mapSize; i++) {
- LazyPrimitive,?> lazyKey = uncheckedGetKey(i);
- if (lazyKey == null) continue;
+ LazyPrimitive, ?> lazyKey = uncheckedGetKey(i);
+ if (lazyKey == null) {
+ continue;
+ }
Object key = lazyKey.getObject();
// do not overwrite if there are duplicate keys
if (key != null && !cachedMap.containsKey(key)) {
@@ -316,7 +326,8 @@
/**
* Get the size of the map represented by this LazyMap.
- * @return The size of the map, -1 for NULL map.
+ *
+ * @return The size of the map, -1 for NULL map.
*/
public int getMapSize() {
if (!parsed) {
Index: serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyArray.java
===================================================================
--- serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyArray.java (revision 901519)
+++ serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyArray.java (working copy)
@@ -28,8 +28,8 @@
/**
* LazyArray stores an array of Lazy Objects.
*
- * LazyArray does not deal with the case of a NULL array. That is handled
- * by the parent LazyObject.
+ * LazyArray does not deal with the case of a NULL array. That is handled by the
+ * parent LazyObject.
*/
public class LazyArray extends LazyNonPrimitive {
@@ -38,36 +38,36 @@
*/
boolean parsed = false;
/**
- * The length of the array.
- * Only valid when the data is parsed.
- * -1 when the array is NULL.
+ * The length of the array. Only valid when the data is parsed. -1 when the
+ * array is NULL.
*/
int arrayLength = 0;
-
+
/**
- * The start positions of array elements.
- * Only valid when the data is parsed.
- * Note that startPosition[arrayLength] = begin + length + 1;
- * that makes sure we can use the same formula to compute the
- * length of each element of the array.
+ * The start positions of array elements. Only valid when the data is parsed.
+ * Note that startPosition[arrayLength] = begin + length + 1; that makes sure
+ * we can use the same formula to compute the length of each element of the
+ * array.
*/
int[] startPosition;
-
+
/**
* Whether init() has been called on the element or not.
*/
boolean[] elementInited;
-
+
/**
- * The elements of the array. Note that we do arrayElements[i].
- * init(bytes, begin, length) only when that element is accessed.
+ * The elements of the array. Note that we do arrayElements[i]. init(bytes,
+ * begin, length) only when that element is accessed.
*/
LazyObject[] arrayElements;
/**
* Construct a LazyArray object with the ObjectInspector.
- * @param oi the oi representing the type of this LazyArray as well as meta
- * information like separator etc.
+ *
+ * @param oi
+ * the oi representing the type of this LazyArray as well as meta
+ * information like separator etc.
*/
protected LazyArray(LazyListObjectInspector oi) {
super(oi);
@@ -75,6 +75,7 @@
/**
* Set the row data for this LazyArray.
+ *
* @see LazyObject#init(ByteArrayRef, int, int)
*/
@Override
@@ -82,84 +83,83 @@
super.init(bytes, start, length);
parsed = false;
}
-
+
/**
- * Enlarge the size of arrays storing information for the elements inside
- * the array.
+ * Enlarge the size of arrays storing information for the elements inside the
+ * array.
*/
private void enlargeArrays() {
if (startPosition == null) {
int initialSize = 2;
- startPosition = new int[initialSize];
+ startPosition = new int[initialSize];
arrayElements = new LazyObject[initialSize];
elementInited = new boolean[initialSize];
} else {
- startPosition = Arrays.copyOf(startPosition, startPosition.length*2);
- arrayElements = Arrays.copyOf(arrayElements, arrayElements.length*2);
- elementInited = Arrays.copyOf(elementInited, elementInited.length*2);
+ startPosition = Arrays.copyOf(startPosition, startPosition.length * 2);
+ arrayElements = Arrays.copyOf(arrayElements, arrayElements.length * 2);
+ elementInited = Arrays.copyOf(elementInited, elementInited.length * 2);
}
}
-
+
/**
* Parse the bytes and fill arrayLength and startPosition.
*/
private void parse() {
parsed = true;
-
+
byte separator = oi.getSeparator();
boolean isEscaped = oi.isEscaped();
byte escapeChar = oi.getEscapeChar();
-
+
// empty array?
if (length == 0) {
arrayLength = 0;
return;
}
-
+
byte[] bytes = this.bytes.getData();
-
+
arrayLength = 0;
int arrayByteEnd = start + length;
int elementByteBegin = start;
int elementByteEnd = start;
-
+
// Go through all bytes in the byte[]
while (elementByteEnd <= arrayByteEnd) {
// Reached the end of a field?
- if (elementByteEnd == arrayByteEnd
- || bytes[elementByteEnd] == separator) {
+ if (elementByteEnd == arrayByteEnd || bytes[elementByteEnd] == separator) {
// Array size not big enough?
- if (startPosition == null || arrayLength+1 == startPosition.length) {
+ if (startPosition == null || arrayLength + 1 == startPosition.length) {
enlargeArrays();
}
startPosition[arrayLength] = elementByteBegin;
arrayLength++;
elementByteBegin = elementByteEnd + 1;
- elementByteEnd ++;
+ elementByteEnd++;
} else {
if (isEscaped && bytes[elementByteEnd] == escapeChar
- && elementByteEnd+1 < arrayByteEnd) {
+ && elementByteEnd + 1 < arrayByteEnd) {
// ignore the char after escape_char
elementByteEnd += 2;
} else {
- elementByteEnd ++;
+ elementByteEnd++;
}
- }
+ }
}
// Store arrayByteEnd+1 in startPosition[arrayLength]
// so that we can use the same formula to compute the length of
// each element in the array: startPosition[i+1] - startPosition[i] - 1
startPosition[arrayLength] = arrayByteEnd + 1;
-
+
if (arrayLength > 0) {
Arrays.fill(elementInited, 0, arrayLength, false);
}
-
+
}
-
+
/**
- * Returns the actual primitive object at the index position
- * inside the array represented by this LazyObject.
+ * Returns the actual primitive object at the index position inside the array
+ * represented by this LazyObject.
*/
public Object getListElementObject(int index) {
if (!parsed) {
@@ -170,34 +170,35 @@
}
return uncheckedGetElement(index);
}
-
+
/**
* Get the element without checking out-of-bound index.
*/
private Object uncheckedGetElement(int index) {
Text nullSequence = oi.getNullSequence();
- int elementLength = startPosition[index+1] - startPosition[index] - 1;
- if (elementLength == nullSequence.getLength()
- && 0 == LazyUtils.compare(bytes.getData(), startPosition[index],
- elementLength, nullSequence.getBytes(), 0,
- nullSequence.getLength())) {
+ int elementLength = startPosition[index + 1] - startPosition[index] - 1;
+ if (elementLength == nullSequence.getLength()
+ && 0 == LazyUtils
+ .compare(bytes.getData(), startPosition[index], elementLength,
+ nullSequence.getBytes(), 0, nullSequence.getLength())) {
return null;
} else {
if (!elementInited[index]) {
elementInited[index] = true;
if (arrayElements[index] == null) {
- arrayElements[index] = LazyFactory.createLazyObject(
- ((ListObjectInspector)oi).getListElementObjectInspector());
+ arrayElements[index] = LazyFactory
+ .createLazyObject(((ListObjectInspector) oi)
+ .getListElementObjectInspector());
}
- arrayElements[index].init(bytes, startPosition[index],
- elementLength);
+ arrayElements[index].init(bytes, startPosition[index], elementLength);
}
}
return arrayElements[index].getObject();
}
-
- /** Returns -1 for null array.
+
+ /**
+ * Returns -1 for null array.
*/
public int getListLength() {
if (!parsed) {
@@ -205,14 +206,15 @@
}
return arrayLength;
}
-
- /**
- * cachedList is reused every time getList is called.
- * Different LazyArray instances cannot share the same cachedList.
+
+ /**
+ * cachedList is reused every time getList is called. Different LazyArray
+ * instances cannot share the same cachedList.
*/
ArrayList cachedList;
- /** Returns the List of actual primitive objects.
- * Returns null for null array.
+
+ /**
+ * Returns the List of actual primitive objects. Returns null for null array.
*/
public List getList() {
if (!parsed) {
@@ -226,7 +228,7 @@
} else {
cachedList.clear();
}
- for (int index=0; index
*
*/
-public class LazyBoolean extends LazyPrimitive {
+public class LazyBoolean extends
+ LazyPrimitive {
public LazyBoolean(LazyBooleanObjectInspector oi) {
super(oi);
@@ -44,28 +45,26 @@
super(copy);
data = new BooleanWritable(copy.data.get());
}
-
+
@Override
public void init(ByteArrayRef bytes, int start, int length) {
- if (length == 4
- && Character.toUpperCase(bytes.getData()[start]) == 'T'
- && Character.toUpperCase(bytes.getData()[start+1]) == 'R'
- && Character.toUpperCase(bytes.getData()[start+2]) == 'U'
- && Character.toUpperCase(bytes.getData()[start+3]) == 'E') {
+ if (length == 4 && Character.toUpperCase(bytes.getData()[start]) == 'T'
+ && Character.toUpperCase(bytes.getData()[start + 1]) == 'R'
+ && Character.toUpperCase(bytes.getData()[start + 2]) == 'U'
+ && Character.toUpperCase(bytes.getData()[start + 3]) == 'E') {
data.set(true);
isNull = false;
} else if (length == 5
- && Character.toUpperCase(bytes.getData()[start]) == 'F'
- && Character.toUpperCase(bytes.getData()[start+1]) == 'A'
- && Character.toUpperCase(bytes.getData()[start+2]) == 'L'
- && Character.toUpperCase(bytes.getData()[start+3]) == 'S'
- && Character.toUpperCase(bytes.getData()[start+4]) == 'E') {
+ && Character.toUpperCase(bytes.getData()[start]) == 'F'
+ && Character.toUpperCase(bytes.getData()[start + 1]) == 'A'
+ && Character.toUpperCase(bytes.getData()[start + 2]) == 'L'
+ && Character.toUpperCase(bytes.getData()[start + 3]) == 'S'
+ && Character.toUpperCase(bytes.getData()[start + 4]) == 'E') {
data.set(false);
isNull = false;
- } else {
+ } else {
isNull = true;
}
}
-
}
Index: serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyNonPrimitive.java
===================================================================
--- serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyNonPrimitive.java (revision 901519)
+++ serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyNonPrimitive.java (working copy)
@@ -18,12 +18,12 @@
package org.apache.hadoop.hive.serde2.lazy;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
-import org.apache.hadoop.io.WritableComparator;
/**
* LazyPrimitive stores a primitive Object in a LazyObject.
*/
-public abstract class LazyNonPrimitive extends LazyObject {
+public abstract class LazyNonPrimitive extends
+ LazyObject {
protected ByteArrayRef bytes;
protected int start;
@@ -31,12 +31,14 @@
/**
* Create a LazyNonPrimitive object with the specified ObjectInspector.
- * @param oi The ObjectInspector would have to have a hierarchy of
- * LazyObjectInspectors with the leaf nodes being
- * WritableObjectInspectors. It's used both for accessing the
- * type hierarchy of the complex object, as well as getting
- * meta information (separator, nullSequence, etc) when parsing
- * the lazy object.
+ *
+ * @param oi
+ * The ObjectInspector would have to have a hierarchy of
+ * LazyObjectInspectors with the leaf nodes being
+ * WritableObjectInspectors. It's used both for accessing the type
+ * hierarchy of the complex object, as well as getting meta
+ * information (separator, nullSequence, etc) when parsing the lazy
+ * object.
*/
protected LazyNonPrimitive(OI oi) {
super(oi);
@@ -44,7 +46,7 @@
start = 0;
length = 0;
}
-
+
@Override
public void init(ByteArrayRef bytes, int start, int length) {
if (bytes == null) {
@@ -59,9 +61,10 @@
@Override
public Object getObject() {
- return this;
+ return this;
}
-
+
+ @Override
public int hashCode() {
return LazyUtils.hashBytes(bytes.getData(), start, length);
}
Index: serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyStruct.java
===================================================================
--- serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyStruct.java (revision 901519)
+++ serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyStruct.java (working copy)
@@ -28,15 +28,15 @@
import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
import org.apache.hadoop.io.Text;
-
/**
- * LazyObject for storing a struct.
- * The field of a struct can be primitive or non-primitive.
- *
- * LazyStruct does not deal with the case of a NULL struct. That is handled
- * by the parent LazyObject.
+ * LazyObject for storing a struct. The field of a struct can be primitive or
+ * non-primitive.
+ *
+ * LazyStruct does not deal with the case of a NULL struct. That is handled by
+ * the parent LazyObject.
*/
-public class LazyStruct extends LazyNonPrimitive {
+public class LazyStruct extends
+ LazyNonPrimitive {
private static Log LOG = LogFactory.getLog(LazyStruct.class.getName());
@@ -46,14 +46,13 @@
boolean parsed;
/**
- * The start positions of struct fields.
- * Only valid when the data is parsed.
- * Note that startPosition[arrayLength] = begin + length + 1;
- * that makes sure we can use the same formula to compute the
- * length of each element of the array.
+ * The start positions of struct fields. Only valid when the data is parsed.
+ * Note that startPosition[arrayLength] = begin + length + 1; that makes sure
+ * we can use the same formula to compute the length of each element of the
+ * array.
*/
int[] startPosition;
-
+
/**
* The fields of the struct.
*/
@@ -62,18 +61,20 @@
* Whether init() has been called on the field or not.
*/
boolean[] fieldInited;
-
+
/**
* Construct a LazyStruct object with the ObjectInspector.
*/
public LazyStruct(LazySimpleStructObjectInspector oi) {
super(oi);
}
-
+
/**
* Set the row data for this LazyStruct.
+ *
* @see LazyObject#init(ByteArrayRef, int, int)
*/
+ @Override
public void init(ByteArrayRef bytes, int start, int length) {
super.init(bytes, start, length);
parsed = false;
@@ -81,34 +82,37 @@
boolean missingFieldWarned = false;
boolean extraFieldWarned = false;
+
/**
* Parse the byte[] and fill each field.
*/
private void parse() {
-
+
byte separator = oi.getSeparator();
boolean lastColumnTakesRest = oi.getLastColumnTakesRest();
boolean isEscaped = oi.isEscaped();
byte escapeChar = oi.getEscapeChar();
-
+
if (fields == null) {
- List extends StructField> fieldRefs = ((StructObjectInspector)oi).getAllStructFieldRefs();
+ List extends StructField> fieldRefs = ((StructObjectInspector) oi)
+ .getAllStructFieldRefs();
fields = new LazyObject[fieldRefs.size()];
- for (int i = 0 ; i < fields.length; i++) {
- fields[i] = LazyFactory.createLazyObject(fieldRefs.get(i).getFieldObjectInspector());
+ for (int i = 0; i < fields.length; i++) {
+ fields[i] = LazyFactory.createLazyObject(fieldRefs.get(i)
+ .getFieldObjectInspector());
}
- fieldInited = new boolean[fields.length];
- // Extra element to make sure we have the same formula to compute the
- // length of each element of the array.
- startPosition = new int[fields.length+1];
+ fieldInited = new boolean[fields.length];
+ // Extra element to make sure we have the same formula to compute the
+ // length of each element of the array.
+ startPosition = new int[fields.length + 1];
}
-
+
int structByteEnd = start + length;
int fieldId = 0;
int fieldByteBegin = start;
int fieldByteEnd = start;
byte[] bytes = this.bytes.getData();
-
+
// Go through all bytes in the byte[]
while (fieldByteEnd <= structByteEnd) {
if (fieldByteEnd == structByteEnd || bytes[fieldByteEnd] == separator) {
@@ -117,7 +121,7 @@
fieldByteEnd = structByteEnd;
}
startPosition[fieldId] = fieldByteBegin;
- fieldId ++;
+ fieldId++;
if (fieldId == fields.length || fieldByteEnd == structByteEnd) {
// All fields have been parsed, or bytes have been parsed.
// We need to set the startPosition of fields.length to ensure we
@@ -127,14 +131,14 @@
// return these fields as NULLs.
for (int i = fieldId; i <= fields.length; i++) {
startPosition[i] = fieldByteEnd + 1;
- }
+ }
break;
}
fieldByteBegin = fieldByteEnd + 1;
- fieldByteEnd ++;
+ fieldByteEnd++;
} else {
if (isEscaped && bytes[fieldByteEnd] == escapeChar
- && fieldByteEnd+1 < structByteEnd) {
+ && fieldByteEnd + 1 < structByteEnd) {
// ignore the char after escape_char
fieldByteEnd += 2;
} else {
@@ -142,36 +146,37 @@
}
}
}
-
+
// Extra bytes at the end?
if (!extraFieldWarned && fieldByteEnd < structByteEnd) {
extraFieldWarned = true;
LOG.warn("Extra bytes detected at the end of the row! Ignoring similar "
+ "problems.");
}
-
+
// Missing fields?
if (!missingFieldWarned && fieldId < fields.length) {
missingFieldWarned = true;
LOG.warn("Missing fields! Expected " + fields.length + " fields but "
+ "only got " + fieldId + "! Ignoring similar problems.");
}
-
+
Arrays.fill(fieldInited, false);
- parsed = true;
+ parsed = true;
}
-
+
/**
* Get one field out of the struct.
*
- * If the field is a primitive field, return the actual object.
- * Otherwise return the LazyObject. This is because PrimitiveObjectInspector
- * does not have control over the object used by the user - the user simply
- * directly use the Object instead of going through
- * Object PrimitiveObjectInspector.get(Object).
+ * If the field is a primitive field, return the actual object. Otherwise
+ * return the LazyObject. This is because PrimitiveObjectInspector does not
+ * have control over the object used by the user - the user simply directly
+ * use the Object instead of going through Object
+ * PrimitiveObjectInspector.get(Object).
*
- * @param fieldID The field ID
- * @return The field as a LazyObject
+ * @param fieldID
+ * The field ID
+ * @return The field as a LazyObject
*/
public Object getField(int fieldID) {
if (!parsed) {
@@ -179,24 +184,27 @@
}
return uncheckedGetField(fieldID);
}
-
+
/**
- * Get the field out of the row without checking parsed.
- * This is called by both getField and getFieldsAsList.
- * @param fieldID The id of the field starting from 0.
- * @param nullSequence The sequence representing NULL value.
- * @return The value of the field
+ * Get the field out of the row without checking parsed. This is called by
+ * both getField and getFieldsAsList.
+ *
+ * @param fieldID
+ * The id of the field starting from 0.
+ * @param nullSequence
+ * The sequence representing NULL value.
+ * @return The value of the field
*/
private Object uncheckedGetField(int fieldID) {
Text nullSequence = oi.getNullSequence();
- // Test the length first so in most cases we avoid doing a byte[]
+ // Test the length first so in most cases we avoid doing a byte[]
// comparison.
int fieldByteBegin = startPosition[fieldID];
- int fieldLength = startPosition[fieldID+1] - startPosition[fieldID] - 1;
+ int fieldLength = startPosition[fieldID + 1] - startPosition[fieldID] - 1;
if ((fieldLength < 0)
- || (fieldLength == nullSequence.getLength()
- && LazyUtils.compare(bytes.getData(), fieldByteBegin, fieldLength,
- nullSequence.getBytes(), 0, nullSequence.getLength()) == 0)) {
+ || (fieldLength == nullSequence.getLength() && LazyUtils.compare(bytes
+ .getData(), fieldByteBegin, fieldLength, nullSequence.getBytes(),
+ 0, nullSequence.getLength()) == 0)) {
return null;
}
if (!fieldInited[fieldID]) {
@@ -207,8 +215,10 @@
}
ArrayList cachedList;
+
/**
* Get the values of the fields as an ArrayList.
+ *
* @return The values of the fields as an ArrayList.
*/
public ArrayList getFieldsAsList() {
@@ -220,12 +230,12 @@
} else {
cachedList.clear();
}
- for (int i=0; i
*
*/
-public class LazyInteger extends LazyPrimitive {
+public class LazyInteger extends
+ LazyPrimitive {
public LazyInteger(LazyIntObjectInspector oi) {
super(oi);
@@ -46,9 +47,8 @@
public LazyInteger(LazyInteger copy) {
super(copy);
data = new IntWritable(copy.data.get());
- }
-
-
+ }
+
@Override
public void init(ByteArrayRef bytes, int start, int length) {
try {
@@ -58,7 +58,7 @@
isNull = true;
}
}
-
+
/**
* Parses the string argument as if it was an int value and returns the
* result. Throws NumberFormatException if the string does not represent an
@@ -67,13 +67,14 @@
* @param bytes
* @param start
* @param length
- * a UTF-8 encoded string representation of an int quantity.
+ * a UTF-8 encoded string representation of an int quantity.
* @return int the value represented by the argument
* @exception NumberFormatException
- * if the argument could not be parsed as an int quantity.
+ * if the argument could not be parsed as an int quantity.
*/
- public static int parseInt(byte[] bytes, int start, int length) throws NumberFormatException {
- return parseInt(bytes, start, length, 10);
+ public static int parseInt(byte[] bytes, int start, int length)
+ throws NumberFormatException {
+ return parseInt(bytes, start, length, 10);
}
/**
@@ -85,20 +86,19 @@
* @param bytes
* @param start
* @param length
- * a UTF-8 encoded string representation of an int quantity.
+ * a UTF-8 encoded string representation of an int quantity.
* @param radix
- * the base to use for conversion.
- * @return the value represented by the argument
+ * the base to use for conversion.
+ * @return the value represented by the argument
* @exception NumberFormatException
- * if the argument could not be parsed as an int quantity.
+ * if the argument could not be parsed as an int quantity.
*/
public static int parseInt(byte[] bytes, int start, int length, int radix)
- throws NumberFormatException {
+ throws NumberFormatException {
if (bytes == null) {
throw new NumberFormatException("String is null");
}
- if (radix < Character.MIN_RADIX ||
- radix > Character.MAX_RADIX) {
+ if (radix < Character.MIN_RADIX || radix > Character.MAX_RADIX) {
throw new NumberFormatException("Invalid radix: " + radix);
}
if (length == 0) {
@@ -109,7 +109,8 @@
if (negative || bytes[start] == '+') {
offset++;
if (length == 1) {
- throw new NumberFormatException(LazyUtils.convertToString(bytes, start, length));
+ throw new NumberFormatException(LazyUtils.convertToString(bytes, start,
+ length));
}
}
@@ -121,57 +122,63 @@
* @param bytes
* @param start
* @param length
- * a UTF-8 encoded string representation of an int quantity.
+ * a UTF-8 encoded string representation of an int quantity.
* @param radix
- * the base to use for conversion.
+ * the base to use for conversion.
* @param offset
- * the starting position after the sign (if exists)
+ * the starting position after the sign (if exists)
* @param radix
- * the base to use for conversion.
+ * the base to use for conversion.
* @param negative
- * whether the number is negative.
+ * whether the number is negative.
* @return the value represented by the argument
* @exception NumberFormatException
- * if the argument could not be parsed as an int quantity.
+ * if the argument could not be parsed as an int quantity.
*/
- private static int parse(byte[] bytes, int start, int length, int offset, int radix,
- boolean negative) throws NumberFormatException {
- int max = Integer.MIN_VALUE / radix;
- int result = 0, end = start + length;
- while (offset < end) {
- int digit = LazyUtils.digit(bytes[offset++], radix);
- if (digit == -1) {
- throw new NumberFormatException(LazyUtils.convertToString(bytes, start, length));
- }
- if (max > result) {
- throw new NumberFormatException(LazyUtils.convertToString(bytes, start, length));
- }
- int next = result * radix - digit;
- if (next > result) {
- throw new NumberFormatException(LazyUtils.convertToString(bytes, start, length));
- }
- result = next;
+ private static int parse(byte[] bytes, int start, int length, int offset,
+ int radix, boolean negative) throws NumberFormatException {
+ int max = Integer.MIN_VALUE / radix;
+ int result = 0, end = start + length;
+ while (offset < end) {
+ int digit = LazyUtils.digit(bytes[offset++], radix);
+ if (digit == -1) {
+ throw new NumberFormatException(LazyUtils.convertToString(bytes, start,
+ length));
}
- if (!negative) {
- result = -result;
- if (result < 0) {
- throw new NumberFormatException(LazyUtils.convertToString(bytes, start, length));
- }
+ if (max > result) {
+ throw new NumberFormatException(LazyUtils.convertToString(bytes, start,
+ length));
}
- return result;
+ int next = result * radix - digit;
+ if (next > result) {
+ throw new NumberFormatException(LazyUtils.convertToString(bytes, start,
+ length));
+ }
+ result = next;
+ }
+ if (!negative) {
+ result = -result;
+ if (result < 0) {
+ throw new NumberFormatException(LazyUtils.convertToString(bytes, start,
+ length));
+ }
+ }
+ return result;
}
/**
- * Writes out the text representation of an integer using base 10 to an
+ * Writes out the text representation of an integer using base 10 to an
* OutputStream in UTF-8 encoding.
- *
- * Note: division by a constant (like 10) is much faster than division by
- * a variable. That's one of the reasons that we don't make radix a
- * parameter here.
- *
- * @param out the outputstream to write to
- * @param i an int to write out
- * @throws IOException
+ *
+ * Note: division by a constant (like 10) is much faster than division by a
+ * variable. That's one of the reasons that we don't make radix a parameter
+ * here.
+ *
+ * @param out
+ * the outputstream to write to
+ * @param i
+ * an int to write out
+ * @throws IOException
*/
public static void writeUTF8(OutputStream out, int i) throws IOException {
if (i == 0) {
@@ -187,23 +194,23 @@
// of overflow here.
i = -i;
}
-
+
int start = 1000000000;
- while (i/start == 0) {
+ while (i / start == 0) {
start /= 10;
}
-
+
while (start > 0) {
out.write('0' - (i / start % 10));
start /= 10;
}
}
-
+
public static void writeUTF8NoException(OutputStream out, int i) {
try {
writeUTF8(out, i);
} catch (IOException e) {
throw new RuntimeException(e);
}
- }
+ }
}
Index: serde/src/java/org/apache/hadoop/hive/serde2/binarysortable/BinarySortableSerDe.java
===================================================================
--- serde/src/java/org/apache/hadoop/hive/serde2/binarysortable/BinarySortableSerDe.java (revision 901519)
+++ serde/src/java/org/apache/hadoop/hive/serde2/binarysortable/BinarySortableSerDe.java (working copy)
@@ -65,43 +65,41 @@
import org.apache.hadoop.io.Writable;
/**
- * BinarySortableSerDe can be used to write data in a way that the data can be
- * compared byte-by-byte with the same order.
+ * BinarySortableSerDe can be used to write data in a way that the data can be
+ * compared byte-by-byte with the same order.
*
- * The data format:
- * NULL: a single byte \0
- * NON-NULL Primitives: ALWAYS prepend a single byte \1, and then:
- * Boolean: FALSE = \1, TRUE = \2
- * Byte: flip the sign-bit to make sure negative comes before positive
- * Short: flip the sign-bit to make sure negative comes before positive
- * Int: flip the sign-bit to make sure negative comes before positive
- * Long: flip the sign-bit to make sure negative comes before positive
- * Double: flip the sign-bit for positive double, and all bits for negative double values
- * String: NULL-terminated UTF-8 string, with NULL escaped to \1 \1, and \1 escaped to \1 \2
- * NON-NULL Complex Types: ALWAYS prepend a single byte \1, and then:
- * Struct: one field by one field.
- * List: \1 followed by each element, and \0 to terminate
- * Map: \1 followed by each key and then each value, and \0 to terminate
- *
- * This SerDe takes an additional parameter SERIALIZATION_SORT_ORDER which is a string containing only "+" and "-".
- * The length of the string should equal to the number of fields in the top-level struct for serialization.
- * "+" means the field should be sorted ascendingly, and "-" means descendingly. The sub fields in the same top-level
- * field will have the same sort order.
+ * The data format: NULL: a single byte \0 NON-NULL Primitives: ALWAYS prepend a
+ * single byte \1, and then: Boolean: FALSE = \1, TRUE = \2 Byte: flip the
+ * sign-bit to make sure negative comes before positive Short: flip the sign-bit
+ * to make sure negative comes before positive Int: flip the sign-bit to make
+ * sure negative comes before positive Long: flip the sign-bit to make sure
+ * negative comes before positive Double: flip the sign-bit for positive double,
+ * and all bits for negative double values String: NULL-terminated UTF-8 string,
+ * with NULL escaped to \1 \1, and \1 escaped to \1 \2 NON-NULL Complex Types:
+ * ALWAYS prepend a single byte \1, and then: Struct: one field by one field.
+ * List: \1 followed by each element, and \0 to terminate Map: \1 followed by
+ * each key and then each value, and \0 to terminate
*
+ * This SerDe takes an additional parameter SERIALIZATION_SORT_ORDER which is a
+ * string containing only "+" and "-". The length of the string should equal to
+ * the number of fields in the top-level struct for serialization. "+" means the
+ * field should be sorted ascendingly, and "-" means descendingly. The sub
+ * fields in the same top-level field will have the same sort order.
+ *
*/
public class BinarySortableSerDe implements SerDe {
- public static final Log LOG = LogFactory.getLog(
- BinarySortableSerDe.class.getName());
-
+ public static final Log LOG = LogFactory.getLog(BinarySortableSerDe.class
+ .getName());
+
List columnNames;
List columnTypes;
-
+
TypeInfo rowTypeInfo;
StructObjectInspector rowObjectInspector;
-
+
boolean[] columnSortOrderIsDesc;
-
+
@Override
public void initialize(Configuration conf, Properties tbl)
throws SerDeException {
@@ -117,27 +115,30 @@
if (columnTypeProperty.length() == 0) {
columnTypes = new ArrayList();
} else {
- columnTypes = TypeInfoUtils.getTypeInfosFromTypeString(columnTypeProperty);
+ columnTypes = TypeInfoUtils
+ .getTypeInfosFromTypeString(columnTypeProperty);
}
- assert(columnNames.size() == columnTypes.size());
-
+ assert (columnNames.size() == columnTypes.size());
+
// Create row related objects
rowTypeInfo = TypeInfoFactory.getStructTypeInfo(columnNames, columnTypes);
- rowObjectInspector = (StructObjectInspector)TypeInfoUtils
+ rowObjectInspector = (StructObjectInspector) TypeInfoUtils
.getStandardWritableObjectInspectorFromTypeInfo(rowTypeInfo);
row = new ArrayList(columnNames.size());
- for (int i=0; i getSerializedClass() {
return BytesWritable.class;
@@ -150,260 +151,268 @@
ArrayList row;
InputByteBuffer inputByteBuffer = new InputByteBuffer();
+
@Override
public Object deserialize(Writable blob) throws SerDeException {
- BytesWritable data = (BytesWritable)blob;
+ BytesWritable data = (BytesWritable) blob;
inputByteBuffer.reset(data.get(), 0, data.getSize());
-
+
try {
- for (int i=0; i r = reuse == null ? new ArrayList() : (ArrayList)reuse;
-
- // Read the list
- int size = 0;
- while (true) {
- int more = buffer.read(invert);
- if (more == 0) {
- // \0 to terminate
- break;
- }
- // \1 followed by each element
- assert(more == 1);
- if (size == r.size()) {
- r.add(null);
- }
- r.set(size, deserialize(buffer, etype, invert, r.get(size)));
- size++;
+ case LONG: {
+ LongWritable r = reuse == null ? new LongWritable()
+ : (LongWritable) reuse;
+ long v = buffer.read(invert) ^ 0x80;
+ for (int i = 0; i < 7; i++) {
+ v = (v << 8) + (buffer.read(invert) & 0xff);
}
- // Remove additional elements if the list is reused
- while (r.size() > size) {
- r.remove(r.size()-1);
+ r.set(v);
+ return r;
+ }
+ case FLOAT: {
+ FloatWritable r = reuse == null ? new FloatWritable()
+ : (FloatWritable) reuse;
+ int v = 0;
+ for (int i = 0; i < 4; i++) {
+ v = (v << 8) + (buffer.read(invert) & 0xff);
}
+ if ((v & (1 << 31)) == 0) {
+ // negative number, flip all bits
+ v = ~v;
+ } else {
+ // positive number, flip the first bit
+ v = v ^ (1 << 31);
+ }
+ r.set(Float.intBitsToFloat(v));
return r;
- }
- case MAP: {
- MapTypeInfo mtype = (MapTypeInfo)type;
- TypeInfo ktype = mtype.getMapKeyTypeInfo();
- TypeInfo vtype = mtype.getMapValueTypeInfo();
-
- // Create the map if needed
- Map r;
- if (reuse == null) {
- r = new HashMap();
+ }
+ case DOUBLE: {
+ DoubleWritable r = reuse == null ? new DoubleWritable()
+ : (DoubleWritable) reuse;
+ long v = 0;
+ for (int i = 0; i < 8; i++) {
+ v = (v << 8) + (buffer.read(invert) & 0xff);
+ }
+ if ((v & (1L << 63)) == 0) {
+ // negative number, flip all bits
+ v = ~v;
} else {
- r = (HashMap)reuse;
- r.clear();
+ // positive number, flip the first bit
+ v = v ^ (1L << 63);
}
-
- // Read the map
- int size = 0;
- while (true) {
- int more = buffer.read(invert);
- if (more == 0) {
- // \0 to terminate
+ r.set(Double.longBitsToDouble(v));
+ return r;
+ }
+ case STRING: {
+ Text r = reuse == null ? new Text() : (Text) reuse;
+ // Get the actual length first
+ int start = buffer.tell();
+ int length = 0;
+ do {
+ byte b = buffer.read(invert);
+ if (b == 0) {
+ // end of string
break;
}
- // \1 followed by each key and then each value
- assert(more == 1);
- Object k = deserialize(buffer, ktype, invert, null);
- Object v = deserialize(buffer, vtype, invert, null);
- r.put(k, v);
+ if (b == 1) {
+ // the last char is an escape char. read the actual char
+ buffer.read(invert);
+ }
+ length++;
+ } while (true);
+
+ if (length == buffer.tell() - start) {
+ // No escaping happened, so we are already done.
+ r.set(buffer.getData(), start, length);
+ } else {
+ // Escaping happened, we need to copy byte-by-byte.
+ // 1. Set the length first.
+ r.set(buffer.getData(), start, length);
+ // 2. Reset the pointer.
+ buffer.seek(start);
+ // 3. Copy the data.
+ byte[] rdata = r.getBytes();
+ for (int i = 0; i < length; i++) {
+ byte b = buffer.read(invert);
+ if (b == 1) {
+ // The last char is an escape char, read the actual char.
+ // The serialization format escape \0 to \1, and \1 to \2,
+ // to make sure the string is null-terminated.
+ b = (byte) (buffer.read(invert) - 1);
+ }
+ rdata[i] = b;
+ }
+ // 4. Read the null terminator.
+ byte b = buffer.read(invert);
+ assert (b == 0);
}
return r;
}
- case STRUCT: {
- StructTypeInfo stype = (StructTypeInfo)type;
- List fieldTypes = stype.getAllStructFieldTypeInfos();
- int size = fieldTypes.size();
- // Create the struct if needed
- ArrayList r = reuse == null ? new ArrayList(size) : (ArrayList)reuse;
- assert(r.size() <= size);
- // Set the size of the struct
- while (r.size() < size) {
+ default: {
+ throw new RuntimeException("Unrecognized type: "
+ + ptype.getPrimitiveCategory());
+ }
+ }
+ }
+ case LIST: {
+ ListTypeInfo ltype = (ListTypeInfo) type;
+ TypeInfo etype = ltype.getListElementTypeInfo();
+
+ // Create the list if needed
+ ArrayList r = reuse == null ? new ArrayList()
+ : (ArrayList) reuse;
+
+ // Read the list
+ int size = 0;
+ while (true) {
+ int more = buffer.read(invert);
+ if (more == 0) {
+ // \0 to terminate
+ break;
+ }
+ // \1 followed by each element
+ assert (more == 1);
+ if (size == r.size()) {
r.add(null);
}
- // Read one field by one field
- for (int eid = 0; eid < size; eid++) {
- r.set(eid, deserialize(buffer, fieldTypes.get(eid), invert, r.get(eid)));
+ r.set(size, deserialize(buffer, etype, invert, r.get(size)));
+ size++;
+ }
+ // Remove additional elements if the list is reused
+ while (r.size() > size) {
+ r.remove(r.size() - 1);
+ }
+ return r;
+ }
+ case MAP: {
+ MapTypeInfo mtype = (MapTypeInfo) type;
+ TypeInfo ktype = mtype.getMapKeyTypeInfo();
+ TypeInfo vtype = mtype.getMapValueTypeInfo();
+
+ // Create the map if needed
+ Map r;
+ if (reuse == null) {
+ r = new HashMap();
+ } else {
+ r = (HashMap) reuse;
+ r.clear();
+ }
+
+ while (true) {
+ int more = buffer.read(invert);
+ if (more == 0) {
+ // \0 to terminate
+ break;
}
- return r;
+ // \1 followed by each key and then each value
+ assert (more == 1);
+ Object k = deserialize(buffer, ktype, invert, null);
+ Object v = deserialize(buffer, vtype, invert, null);
+ r.put(k, v);
}
- default: {
- throw new RuntimeException("Unrecognized type: " + type.getCategory());
+ return r;
+ }
+ case STRUCT: {
+ StructTypeInfo stype = (StructTypeInfo) type;
+ List fieldTypes = stype.getAllStructFieldTypeInfos();
+ int size = fieldTypes.size();
+ // Create the struct if needed
+ ArrayList r = reuse == null ? new ArrayList(size)
+ : (ArrayList) reuse;
+ assert (r.size() <= size);
+ // Set the size of the struct
+ while (r.size() < size) {
+ r.add(null);
}
+ // Read one field by one field
+ for (int eid = 0; eid < size; eid++) {
+ r
+ .set(eid, deserialize(buffer, fieldTypes.get(eid), invert, r
+ .get(eid)));
+ }
+ return r;
}
+ default: {
+ throw new RuntimeException("Unrecognized type: " + type.getCategory());
+ }
+ }
}
-
+
BytesWritable serializeBytesWritable = new BytesWritable();
OutputByteBuffer outputByteBuffer = new OutputByteBuffer();
-
+
@Override
public Writable serialize(Object obj, ObjectInspector objInspector)
throws SerDeException {
outputByteBuffer.reset();
- StructObjectInspector soi = (StructObjectInspector)objInspector;
+ StructObjectInspector soi = (StructObjectInspector) objInspector;
List extends StructField> fields = soi.getAllStructFieldRefs();
-
- for (int i=0; i> 8) ^ 0x80), invert);
- buffer.write((byte)v, invert);
- return;
- }
- case INT: {
- IntObjectInspector ioi = (IntObjectInspector)poi;
- int v = ioi.get(o);
- buffer.write((byte)((v >> 24) ^ 0x80), invert);
- buffer.write((byte)(v >> 16), invert);
- buffer.write((byte)(v >> 8), invert);
- buffer.write((byte)v, invert);
- return;
- }
- case LONG: {
- LongObjectInspector loi = (LongObjectInspector)poi;
- long v = loi.get(o);
- buffer.write((byte)((v >> 56) ^ 0x80), invert);
- buffer.write((byte)(v >> 48), invert);
- buffer.write((byte)(v >> 40), invert);
- buffer.write((byte)(v >> 32), invert);
- buffer.write((byte)(v >> 24), invert);
- buffer.write((byte)(v >> 16), invert);
- buffer.write((byte)(v >> 8), invert);
- buffer.write((byte)v, invert);
- return;
- }
- case FLOAT: {
- FloatObjectInspector foi = (FloatObjectInspector)poi;
- int v = Float.floatToIntBits(foi.get(o));
- if ((v & (1<<31)) != 0) {
- // negative number, flip all bits
- v = ~v;
- } else {
- // positive number, flip the first bit
- v = v ^ (1<<31);
- }
- buffer.write((byte)(v >> 24), invert);
- buffer.write((byte)(v >> 16), invert);
- buffer.write((byte)(v >> 8), invert);
- buffer.write((byte)v, invert);
- return;
- }
- case DOUBLE: {
- DoubleObjectInspector doi = (DoubleObjectInspector)poi;
- long v = Double.doubleToLongBits(doi.get(o));
- if ((v & (1L<<63)) != 0) {
- // negative number, flip all bits
- v = ~v;
- } else {
- // positive number, flip the first bit
- v = v ^ (1L<<63);
- }
- buffer.write((byte)(v >> 56), invert);
- buffer.write((byte)(v >> 48), invert);
- buffer.write((byte)(v >> 40), invert);
- buffer.write((byte)(v >> 32), invert);
- buffer.write((byte)(v >> 24), invert);
- buffer.write((byte)(v >> 16), invert);
- buffer.write((byte)(v >> 8), invert);
- buffer.write((byte)v, invert);
- return;
- }
- case STRING: {
- StringObjectInspector soi = (StringObjectInspector)poi;
- Text t = soi.getPrimitiveWritableObject(o);
- byte[] data = t.getBytes();
- int length = t.getLength();
- for (int i=0; i> 8) ^ 0x80), invert);
+ buffer.write((byte) v, invert);
+ return;
+ }
+ case INT: {
+ IntObjectInspector ioi = (IntObjectInspector) poi;
+ int v = ioi.get(o);
+ buffer.write((byte) ((v >> 24) ^ 0x80), invert);
+ buffer.write((byte) (v >> 16), invert);
+ buffer.write((byte) (v >> 8), invert);
+ buffer.write((byte) v, invert);
+ return;
+ }
+ case LONG: {
+ LongObjectInspector loi = (LongObjectInspector) poi;
+ long v = loi.get(o);
+ buffer.write((byte) ((v >> 56) ^ 0x80), invert);
+ buffer.write((byte) (v >> 48), invert);
+ buffer.write((byte) (v >> 40), invert);
+ buffer.write((byte) (v >> 32), invert);
+ buffer.write((byte) (v >> 24), invert);
+ buffer.write((byte) (v >> 16), invert);
+ buffer.write((byte) (v >> 8), invert);
+ buffer.write((byte) v, invert);
+ return;
+ }
+ case FLOAT: {
+ FloatObjectInspector foi = (FloatObjectInspector) poi;
+ int v = Float.floatToIntBits(foi.get(o));
+ if ((v & (1 << 31)) != 0) {
+ // negative number, flip all bits
+ v = ~v;
+ } else {
+ // positive number, flip the first bit
+ v = v ^ (1 << 31);
}
- // and \0 to terminate
- buffer.write((byte)0, invert);
+ buffer.write((byte) (v >> 24), invert);
+ buffer.write((byte) (v >> 16), invert);
+ buffer.write((byte) (v >> 8), invert);
+ buffer.write((byte) v, invert);
return;
- }
- case MAP: {
- MapObjectInspector moi = (MapObjectInspector)oi;
- ObjectInspector koi = moi.getMapKeyObjectInspector();
- ObjectInspector voi = moi.getMapValueObjectInspector();
-
- // \1 followed by each key and then each value
- Map, ?> map = moi.getMap(o);
- for(Map.Entry, ?> entry: map.entrySet()) {
- buffer.write((byte)1, invert);
- serialize(buffer, entry.getKey(), koi, invert);
- serialize(buffer, entry.getValue(), voi, invert);
+ }
+ case DOUBLE: {
+ DoubleObjectInspector doi = (DoubleObjectInspector) poi;
+ long v = Double.doubleToLongBits(doi.get(o));
+ if ((v & (1L << 63)) != 0) {
+ // negative number, flip all bits
+ v = ~v;
+ } else {
+ // positive number, flip the first bit
+ v = v ^ (1L << 63);
}
- // and \0 to terminate
- buffer.write((byte)0, invert);
+ buffer.write((byte) (v >> 56), invert);
+ buffer.write((byte) (v >> 48), invert);
+ buffer.write((byte) (v >> 40), invert);
+ buffer.write((byte) (v >> 32), invert);
+ buffer.write((byte) (v >> 24), invert);
+ buffer.write((byte) (v >> 16), invert);
+ buffer.write((byte) (v >> 8), invert);
+ buffer.write((byte) v, invert);
return;
}
- case STRUCT: {
- StructObjectInspector soi = (StructObjectInspector)oi;
- List extends StructField> fields = soi.getAllStructFieldRefs();
-
- for (int i=0; i map = moi.getMap(o);
+ for (Map.Entry, ?> entry : map.entrySet()) {
+ buffer.write((byte) 1, invert);
+ serialize(buffer, entry.getKey(), koi, invert);
+ serialize(buffer, entry.getValue(), voi, invert);
+ }
+ // and \0 to terminate
+ buffer.write((byte) 0, invert);
+ return;
+ }
+ case STRUCT: {
+ StructObjectInspector soi = (StructObjectInspector) oi;
+ List extends StructField> fields = soi.getAllStructFieldRefs();
+
+ for (int i = 0; i < fields.size(); i++) {
+ serialize(buffer, soi.getStructFieldData(o, fields.get(i)), fields.get(
+ i).getFieldObjectInspector(), invert);
+ }
+ return;
+ }
+ default: {
+ throw new RuntimeException("Unrecognized type: " + oi.getCategory());
+ }
+ }
+
}
}
Index: serde/src/java/org/apache/hadoop/hive/serde2/binarysortable/InputByteBuffer.java
===================================================================
--- serde/src/java/org/apache/hadoop/hive/serde2/binarysortable/InputByteBuffer.java (revision 901519)
+++ serde/src/java/org/apache/hadoop/hive/serde2/binarysortable/InputByteBuffer.java (working copy)
@@ -21,15 +21,15 @@
import java.io.IOException;
/**
- * This class is much more efficient than ByteArrayInputStream
- * because none of the methods are synchronized.
+ * This class is much more efficient than ByteArrayInputStream because none of
+ * the methods are synchronized.
*/
public class InputByteBuffer {
-
+
byte[] data;
int start;
int end;
-
+
/**
* Reset the byte buffer to the given byte range.
*/
@@ -38,66 +38,63 @@
this.start = start;
this.end = end;
}
-
+
public final byte read() throws IOException {
return read(false);
}
-
+
/**
- * Read one byte from the byte buffer.
- * Final method to help inlining.
- * @param invert whether we want to invert all the bits.
+ * Read one byte from the byte buffer. Final method to help inlining.
+ *
+ * @param invert
+ * whether we want to invert all the bits.
*/
public final byte read(boolean invert) throws IOException {
if (start >= end) {
throw new EOFException();
}
if (invert) {
- return (byte)(0xff ^ data[start++]);
+ return (byte) (0xff ^ data[start++]);
} else {
return data[start++];
}
}
-
+
/**
- * Return the current position.
- * Final method to help inlining.
+ * Return the current position. Final method to help inlining.
*/
public final int tell() {
return start;
}
-
+
/**
- * Set the current position.
- * Final method to help inlining.
+ * Set the current position. Final method to help inlining.
*/
public final void seek(int position) {
start = position;
}
-
+
public final int getEnd() {
return end;
}
-
+
/**
* Returns the underlying byte array.
*/
public final byte[] getData() {
return data;
}
-
+
/**
* Return the bytes in hex format.
*/
public String dumpHex() {
StringBuilder sb = new StringBuilder();
- for (int i=start; i notSkipIDs = ColumnProjectionUtils.getReadColumnIDs(job);
-
+ java.util.ArrayList notSkipIDs = ColumnProjectionUtils
+ .getReadColumnIDs(job);
+
cachedLazyStruct = new ColumnarStruct(cachedObjectInspector, notSkipIDs);
-
+
int size = serdeParams.getColumnTypes().size();
field = new BytesRefWritable[size];
for (int i = 0; i < size; i++) {
@@ -204,17 +205,17 @@
&& (declaredFields == null || declaredFields.get(i)
.getFieldObjectInspector().getCategory().equals(
Category.PRIMITIVE))) {
- LazySimpleSerDe.serialize(serializeStream,
- SerDeUtils.getJSONString(f, foi),
+ LazySimpleSerDe.serialize(serializeStream, SerDeUtils.getJSONString(
+ f, foi),
PrimitiveObjectInspectorFactory.javaStringObjectInspector,
serdeParams.getSeparators(), 1, serdeParams.getNullSequence(),
- serdeParams.isEscaped(), serdeParams.getEscapeChar(),
- serdeParams.getNeedsEscape());
+ serdeParams.isEscaped(), serdeParams.getEscapeChar(), serdeParams
+ .getNeedsEscape());
} else {
- LazySimpleSerDe.serialize(serializeStream, f, foi,
- serdeParams.getSeparators(), 1, serdeParams.getNullSequence(),
- serdeParams.isEscaped(), serdeParams.getEscapeChar(),
- serdeParams.getNeedsEscape());
+ LazySimpleSerDe.serialize(serializeStream, f, foi, serdeParams
+ .getSeparators(), 1, serdeParams.getNullSequence(), serdeParams
+ .isEscaped(), serdeParams.getEscapeChar(), serdeParams
+ .getNeedsEscape());
}
field[i].set(serializeStream.getData(), count, serializeStream
Index: serde/src/java/org/apache/hadoop/hive/serde2/columnar/ColumnarStruct.java
===================================================================
--- serde/src/java/org/apache/hadoop/hive/serde2/columnar/ColumnarStruct.java (revision 901519)
+++ serde/src/java/org/apache/hadoop/hive/serde2/columnar/ColumnarStruct.java (working copy)
@@ -20,9 +20,10 @@
import java.io.IOException;
import java.util.ArrayList;
-import java.util.Collections;
import java.util.List;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hive.serde2.lazy.ByteArrayRef;
import org.apache.hadoop.hive.serde2.lazy.LazyFactory;
import org.apache.hadoop.hive.serde2.lazy.LazyObject;
@@ -31,8 +32,6 @@
import org.apache.hadoop.hive.serde2.objectinspector.StructField;
import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
import org.apache.hadoop.io.Text;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
/**
* ColumnarStruct is different from LazyStruct in that ColumnarStruct's field
@@ -47,11 +46,11 @@
* The fields of the struct.
*/
LazyObject[] fields;
-
+
private static final Log LOG = LogFactory.getLog(ColumnarStruct.class);
-
- int[] prjColIDs = null; // list of projected column IDs
-
+
+ int[] prjColIDs = null; // list of projected column IDs
+
/**
* Construct a ColumnarStruct object with the TypeInfo. It creates the first
* level object at the first place
@@ -62,7 +61,7 @@
public ColumnarStruct(ObjectInspector oi) {
this(oi, null);
}
-
+
/**
* Construct a ColumnarStruct object with the TypeInfo. It creates the first
* level object at the first place
@@ -70,44 +69,50 @@
* @param oi
* the ObjectInspector representing the type of this LazyStruct.
* @param notSkippedColumnIDs
- * the column ids that should not be skipped
+ * the column ids that should not be skipped
*/
- public ColumnarStruct(ObjectInspector oi, ArrayList notSkippedColumnIDs) {
- List extends StructField> fieldRefs = ((StructObjectInspector) oi).getAllStructFieldRefs();
+ public ColumnarStruct(ObjectInspector oi,
+ ArrayList notSkippedColumnIDs) {
+ List extends StructField> fieldRefs = ((StructObjectInspector) oi)
+ .getAllStructFieldRefs();
int num = fieldRefs.size();
fields = new LazyObject[num];
cachedByteArrayRef = new ByteArrayRef[num];
rawBytesField = new BytesRefWritable[num];
fieldSkipped = new boolean[num];
inited = new boolean[num];
-
- //if no columns is set to be skipped, add all columns in 'notSkippedColumnIDs'
- if (notSkippedColumnIDs == null || notSkippedColumnIDs.size() == 0) {
- for (int i = 0; i < num; i++)
- notSkippedColumnIDs.add(i);
- }
-
+
+ // if no columns is set to be skipped, add all columns in
+ // 'notSkippedColumnIDs'
+ if (notSkippedColumnIDs == null || notSkippedColumnIDs.size() == 0) {
+ for (int i = 0; i < num; i++) {
+ notSkippedColumnIDs.add(i);
+ }
+ }
+
for (int i = 0; i < num; i++) {
- fields[i] = LazyFactory.createLazyObject(fieldRefs.get(i).getFieldObjectInspector());
+ fields[i] = LazyFactory.createLazyObject(fieldRefs.get(i)
+ .getFieldObjectInspector());
cachedByteArrayRef[i] = new ByteArrayRef();
- if(!notSkippedColumnIDs.contains(i)){
- fieldSkipped[i] = true;
- inited[i] = true;
- } else
- inited[i] = false;
+ if (!notSkippedColumnIDs.contains(i)) {
+ fieldSkipped[i] = true;
+ inited[i] = true;
+ } else {
+ inited[i] = false;
+ }
}
-
- // maintain a list of non-NULL column IDs
- int min = notSkippedColumnIDs.size() > num ? num : notSkippedColumnIDs
- .size();
- prjColIDs = new int[min];
- for (int i = 0, index = 0; i < notSkippedColumnIDs.size(); ++i) {
- int readCol = notSkippedColumnIDs.get(i).intValue();
- if (readCol < num) {
- prjColIDs[index] = readCol;
- index++;
- }
- }
+
+ // maintain a list of non-NULL column IDs
+ int min = notSkippedColumnIDs.size() > num ? num : notSkippedColumnIDs
+ .size();
+ prjColIDs = new int[min];
+ for (int i = 0, index = 0; i < notSkippedColumnIDs.size(); ++i) {
+ int readCol = notSkippedColumnIDs.get(i).intValue();
+ if (readCol < num) {
+ prjColIDs[index] = readCol;
+ index++;
+ }
+ }
}
/**
@@ -152,8 +157,9 @@
* @return The value of the field
*/
protected Object uncheckedGetField(int fieldID, Text nullSequence) {
- if (fieldSkipped[fieldID])
+ if (fieldSkipped[fieldID]) {
return null;
+ }
if (!inited[fieldID]) {
BytesRefWritable passedInField = rawBytesField[fieldID];
try {
@@ -165,35 +171,36 @@
.getStart(), passedInField.getLength());
inited[fieldID] = true;
}
-
+
byte[] data = cachedByteArrayRef[fieldID].getData();
int fieldLen = rawBytesField[fieldID].length;
-
+
if (fieldLen == nullSequence.getLength()
- && LazyUtils.compare(data, rawBytesField[fieldID].getStart(),
- fieldLen, nullSequence.getBytes(), 0, nullSequence.getLength()) == 0) {
+ && LazyUtils.compare(data, rawBytesField[fieldID].getStart(), fieldLen,
+ nullSequence.getBytes(), 0, nullSequence.getLength()) == 0) {
return null;
}
return fields[fieldID].getObject();
}
-
- /* ============================ [PERF] ===================================
- * This function is called for every row. Setting up the selected/projected
- * columns at the first call, and don't do that for the following calls.
- * Ideally this should be done in the constructor where we don't need to
- * branch in the function for each row.
- * =========================================================================
+
+ /*
+ * ============================ [PERF] ===================================
+ * This function is called for every row. Setting up the selected/projected
+ * columns at the first call, and don't do that for the following calls.
+ * Ideally this should be done in the constructor where we don't need to
+ * branch in the function for each row.
+ * =========================================================================
*/
public void init(BytesRefArrayWritable cols) {
- for (int i = 0; i < prjColIDs.length; ++i ) {
+ for (int i = 0; i < prjColIDs.length; ++i) {
int fieldIndex = prjColIDs[i];
- if(fieldIndex < cols.size()){
- rawBytesField[fieldIndex] = cols.unCheckedGet(fieldIndex);
- inited[fieldIndex] = false;
+ if (fieldIndex < cols.size()) {
+ rawBytesField[fieldIndex] = cols.unCheckedGet(fieldIndex);
+ inited[fieldIndex] = false;
} else {
- // select columns that actually do not exist in the file.
- fieldSkipped[fieldIndex] = true;
+ // select columns that actually do not exist in the file.
+ fieldSkipped[fieldIndex] = true;
}
}
}
Index: serde/src/java/org/apache/hadoop/hive/serde2/columnar/LazyDecompressionCallback.java
===================================================================
--- serde/src/java/org/apache/hadoop/hive/serde2/columnar/LazyDecompressionCallback.java (revision 901519)
+++ serde/src/java/org/apache/hadoop/hive/serde2/columnar/LazyDecompressionCallback.java (working copy)
@@ -20,9 +20,8 @@
import java.io.IOException;
-
/**
- * Used to call back lazy decompression process.
+ * Used to call back lazy decompression process.
*
* @see #BytesRefWritable
*/
Index: serde/src/java/org/apache/hadoop/hive/serde2/columnar/BytesRefArrayWritable.java
===================================================================
--- serde/src/java/org/apache/hadoop/hive/serde2/columnar/BytesRefArrayWritable.java (revision 901519)
+++ serde/src/java/org/apache/hadoop/hive/serde2/columnar/BytesRefArrayWritable.java (working copy)
@@ -54,8 +54,9 @@
* if the specified initial capacity is negative
*/
public BytesRefArrayWritable(int capacity) {
- if (capacity < 0)
+ if (capacity < 0) {
throw new IllegalArgumentException("Capacity can not be negative.");
+ }
bytesRefWritables = new BytesRefWritable[0];
ensureCapacity(capacity);
}
@@ -85,10 +86,10 @@
* @throws IndexOutOfBoundsException
*/
public BytesRefWritable get(int index) {
- if (index >= valid)
+ if (index >= valid) {
throw new IndexOutOfBoundsException(
- "This BytesRefArrayWritable only has " + valid
- + " valid values.");
+ "This BytesRefArrayWritable only has " + valid + " valid values.");
+ }
return bytesRefWritables[index];
}
@@ -115,12 +116,14 @@
* if the specified new element is null
*/
public void set(int index, BytesRefWritable bytesRefWritable) {
- if (bytesRefWritable == null)
+ if (bytesRefWritable == null) {
throw new IllegalArgumentException("Can not assign null.");
+ }
ensureCapacity(index + 1);
bytesRefWritables[index] = bytesRefWritable;
- if (valid <= index)
+ if (valid <= index) {
valid = index + 1;
+ }
}
/**
@@ -128,18 +131,22 @@
*/
@Override
public int compareTo(BytesRefArrayWritable other) {
- if (other == null)
+ if (other == null) {
throw new IllegalArgumentException("Argument can not be null.");
- if (this == other)
+ }
+ if (this == other) {
return 0;
+ }
int sizeDiff = valid - other.valid;
- if (sizeDiff != 0)
+ if (sizeDiff != 0) {
return sizeDiff;
+ }
for (int i = 0; i < valid; i++) {
- if (other.contains(bytesRefWritables[i]))
+ if (other.contains(bytesRefWritables[i])) {
continue;
- else
+ } else {
return 1;
+ }
}
return 0;
}
@@ -155,11 +162,13 @@
* if the specified element is null
*/
public boolean contains(BytesRefWritable bytesRefWritable) {
- if (bytesRefWritable == null)
+ if (bytesRefWritable == null) {
throw new IllegalArgumentException("Argument can not be null.");
+ }
for (int i = 0; i < valid; i++) {
- if (bytesRefWritables[i].equals(bytesRefWritable))
+ if (bytesRefWritables[i].equals(bytesRefWritable)) {
return true;
+ }
}
return false;
}
@@ -167,9 +176,11 @@
/**
* {@inheritDoc}
*/
+ @Override
public boolean equals(Object o) {
- if (o == null || !(o instanceof BytesRefArrayWritable))
+ if (o == null || !(o instanceof BytesRefArrayWritable)) {
return false;
+ }
return compareTo((BytesRefArrayWritable) o) == 0;
}
Index: serde/src/java/org/apache/hadoop/hive/serde2/columnar/BytesRefWritable.java
===================================================================
--- serde/src/java/org/apache/hadoop/hive/serde2/columnar/BytesRefWritable.java (revision 901519)
+++ serde/src/java/org/apache/hadoop/hive/serde2/columnar/BytesRefWritable.java (working copy)
@@ -39,9 +39,8 @@
int start = 0;
int length = 0;
byte[] bytes = null;
-
+
LazyDecompressionCallback lazyDecompressObj;
-
/**
* Create a zero-size bytes.
@@ -78,7 +77,7 @@
start = offset;
length = len;
}
-
+
/**
* Create a BytesRefWritable referenced to one section of the given bytes. The
* argument lazyDecompressData refers to a LazyDecompressionCallback
@@ -86,7 +85,8 @@
* uncompressed bytes of lazyDecompressData . Use offset and
* len after uncompressing the data.
*/
- public BytesRefWritable(LazyDecompressionCallback lazyDecompressData, int offset, int len) {
+ public BytesRefWritable(LazyDecompressionCallback lazyDecompressData,
+ int offset, int len) {
lazyDecompressObj = lazyDecompressData;
start = offset;
length = len;
@@ -102,7 +102,7 @@
* Returns a copy of the underlying bytes referenced by this instance.
*
* @return a new copied byte array
- * @throws IOException
+ * @throws IOException
*/
public byte[] getBytesCopy() throws IOException {
lazyDecompress();
@@ -113,7 +113,8 @@
/**
* Returns the underlying bytes.
- * @throws IOException
+ *
+ * @throws IOException
*/
public byte[] getData() throws IOException {
lazyDecompress();
@@ -132,7 +133,7 @@
length = len;
lazyDecompressObj = null;
}
-
+
/**
* readFields() will corrupt the array. So use the set method whenever
* possible.
@@ -176,11 +177,13 @@
}
/** {@inheritDoc} */
+ @Override
public int hashCode() {
return super.hashCode();
}
/** {@inheritDoc} */
+ @Override
public String toString() {
StringBuffer sb = new StringBuffer(3 * length);
for (int idx = start; idx < length; idx++) {
@@ -201,22 +204,26 @@
/** {@inheritDoc} */
@Override
public int compareTo(BytesRefWritable other) {
- if (other == null)
+ if (other == null) {
throw new IllegalArgumentException("Argument can not be null.");
- if (this == other)
+ }
+ if (this == other) {
return 0;
+ }
try {
- return WritableComparator.compareBytes(getData(), start, getLength(), other
- .getData(), other.start, other.getLength());
+ return WritableComparator.compareBytes(getData(), start, getLength(),
+ other.getData(), other.start, other.getLength());
} catch (IOException e) {
throw new RuntimeException(e);
}
}
/** {@inheritDoc} */
+ @Override
public boolean equals(Object right_obj) {
- if (right_obj == null || !(right_obj instanceof BytesRefWritable))
+ if (right_obj == null || !(right_obj instanceof BytesRefWritable)) {
return false;
+ }
return compareTo((BytesRefWritable) right_obj) == 0;
}
Index: serde/src/java/org/apache/hadoop/hive/serde2/SerDe.java
===================================================================
--- serde/src/java/org/apache/hadoop/hive/serde2/SerDe.java (revision 901519)
+++ serde/src/java/org/apache/hadoop/hive/serde2/SerDe.java (working copy)
@@ -21,12 +21,12 @@
/**
* A union of HiveDeserializer and HiveSerializer interface.
*
- * If a developer wants his hive table to be read-only, then he just want to
- * return
+ * If a developer wants his hive table to be read-only, then he just want to
+ * return
*
* both readable and writable, then
- *
- *
+ *
+ *
*/
public interface SerDe extends Deserializer, Serializer {
Index: serde/src/java/org/apache/hadoop/hive/serde2/MetadataTypedColumnsetSerDe.java
===================================================================
--- serde/src/java/org/apache/hadoop/hive/serde2/MetadataTypedColumnsetSerDe.java (revision 901519)
+++ serde/src/java/org/apache/hadoop/hive/serde2/MetadataTypedColumnsetSerDe.java (working copy)
@@ -38,35 +38,38 @@
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
-
public class MetadataTypedColumnsetSerDe implements SerDe {
- public static final Log LOG = LogFactory.getLog(MetadataTypedColumnsetSerDe.class.getName());
+ public static final Log LOG = LogFactory
+ .getLog(MetadataTypedColumnsetSerDe.class.getName());
static {
StackTraceElement[] sTrace = new Exception().getStackTrace();
String className = sTrace[0].getClassName();
try {
- // For backward compatibility: this class replaces the columnsetSerDe class.
- SerDeUtils.registerSerDe("org.apache.hadoop.hive.serde.thrift.columnsetSerDe",
- Class.forName(className));
- } catch(Exception e) {
+ // For backward compatibility: this class replaces the columnsetSerDe
+ // class.
+ SerDeUtils.registerSerDe(
+ "org.apache.hadoop.hive.serde.thrift.columnsetSerDe", Class
+ .forName(className));
+ } catch (Exception e) {
throw new RuntimeException(e);
}
}
-
+
final public static String DefaultSeparator = "\001";
private String separator;
final public static String defaultNullString = "\\N";
- private String nullString;
+ private String nullString;
private List columnNames;
private ObjectInspector cachedObjectInspector;
private boolean lastColumnTakesRest = false;
private int splitLimit = -1;
-
+
+ @Override
public String toString() {
return "MetaDataTypedColumnsetSerDe[" + separator + "," + columnNames + "]";
}
@@ -78,17 +81,18 @@
private String getByteValue(String altValue, String defaultVal) {
if (altValue != null && altValue.length() > 0) {
try {
- byte b [] = new byte[1];
+ byte b[] = new byte[1];
b[0] = Byte.valueOf(altValue).byteValue();
return new String(b);
- } catch(NumberFormatException e) {
+ } catch (NumberFormatException e) {
return altValue;
}
}
return defaultVal;
}
- public void initialize(Configuration job, Properties tbl) throws SerDeException {
+ public void initialize(Configuration job, Properties tbl)
+ throws SerDeException {
String alt_sep = tbl.getProperty(Constants.SERIALIZATION_FORMAT);
separator = getByteValue(alt_sep, DefaultSeparator);
@@ -97,34 +101,46 @@
String columnProperty = tbl.getProperty("columns");
String serdeName = tbl.getProperty(Constants.SERIALIZATION_LIB);
- // tables that were serialized with columnsetSerDe doesn't have metadata
- // so this hack applies to all such tables
+ // tables that were serialized with columnsetSerDe doesn't have metadata
+ // so this hack applies to all such tables
boolean columnsetSerDe = false;
- if ((serdeName != null) && serdeName.equals("org.apache.hadoop.hive.serde.thrift.columnsetSerDe")) {
+ if ((serdeName != null)
+ && serdeName
+ .equals("org.apache.hadoop.hive.serde.thrift.columnsetSerDe")) {
columnsetSerDe = true;
}
- if (columnProperty == null || columnProperty.length() == 0
+ if (columnProperty == null || columnProperty.length() == 0
|| columnsetSerDe) {
// Hack for tables with no columns
// Treat it as a table with a single column called "col"
- cachedObjectInspector = ObjectInspectorFactory.getReflectionObjectInspector(
- ColumnSet.class, ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
+ cachedObjectInspector = ObjectInspectorFactory
+ .getReflectionObjectInspector(ColumnSet.class,
+ ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
} else {
columnNames = Arrays.asList(columnProperty.split(","));
- cachedObjectInspector = MetadataListStructObjectInspector.getInstance(columnNames);
+ cachedObjectInspector = MetadataListStructObjectInspector
+ .getInstance(columnNames);
}
-
- String lastColumnTakesRestString = tbl.getProperty(Constants.SERIALIZATION_LAST_COLUMN_TAKES_REST);
- lastColumnTakesRest = (lastColumnTakesRestString != null && lastColumnTakesRestString.equalsIgnoreCase("true"));
- splitLimit = (lastColumnTakesRest && columnNames != null) ? columnNames.size() : -1;
-
- LOG.debug(getClass().getName() + ": initialized with columnNames: " + columnNames + " and separator code=" + (int)separator.charAt(0)
- + " lastColumnTakesRest=" + lastColumnTakesRest + " splitLimit=" + splitLimit);
+
+ String lastColumnTakesRestString = tbl
+ .getProperty(Constants.SERIALIZATION_LAST_COLUMN_TAKES_REST);
+ lastColumnTakesRest = (lastColumnTakesRestString != null && lastColumnTakesRestString
+ .equalsIgnoreCase("true"));
+ splitLimit = (lastColumnTakesRest && columnNames != null) ? columnNames
+ .size() : -1;
+
+ LOG.debug(getClass().getName() + ": initialized with columnNames: "
+ + columnNames + " and separator code=" + (int) separator.charAt(0)
+ + " lastColumnTakesRest=" + lastColumnTakesRest + " splitLimit="
+ + splitLimit);
}
-
+
/**
* Split the row into columns.
- * @param limit up to limit columns will be produced (the last column takes all the rest), -1 for unlimited.
+ *
+ * @param limit
+ * up to limit columns will be produced (the last column takes all
+ * the rest), -1 for unlimited.
* @return The ColumnSet object
* @throws Exception
*/
@@ -135,9 +151,9 @@
} else {
c.col.clear();
}
- String [] l1 = row.split(sep, limit);
+ String[] l1 = row.split(sep, limit);
- for(String s: l1) {
+ for (String s : l1) {
if (s.equals(nullString)) {
c.col.add(null);
} else {
@@ -146,12 +162,13 @@
}
return (c);
}
-
+
ColumnSet deserializeCache = new ColumnSet();
+
public Object deserialize(Writable field) throws SerDeException {
String row = null;
if (field instanceof BytesWritable) {
- BytesWritable b = (BytesWritable)field;
+ BytesWritable b = (BytesWritable) field;
try {
row = Text.decode(b.get(), 0, b.getSize());
} catch (CharacterCodingException e) {
@@ -163,17 +180,17 @@
try {
deserialize(deserializeCache, row, separator, nullString, splitLimit);
if (columnNames != null) {
- assert(columnNames.size() == deserializeCache.col.size());
+ assert (columnNames.size() == deserializeCache.col.size());
}
return deserializeCache;
} catch (ClassCastException e) {
- throw new SerDeException( this.getClass().getName() + " expects Text or BytesWritable", e);
+ throw new SerDeException(this.getClass().getName()
+ + " expects Text or BytesWritable", e);
} catch (Exception e) {
throw new SerDeException(e);
}
}
-
-
+
public ObjectInspector getObjectInspector() throws SerDeException {
return cachedObjectInspector;
}
@@ -181,27 +198,33 @@
public Class extends Writable> getSerializedClass() {
return Text.class;
}
-
+
Text serializeCache = new Text();
- public Writable serialize(Object obj, ObjectInspector objInspector) throws SerDeException {
+ public Writable serialize(Object obj, ObjectInspector objInspector)
+ throws SerDeException {
+
if (objInspector.getCategory() != Category.STRUCT) {
- throw new SerDeException(getClass().toString()
- + " can only serialize struct types, but we got: " + objInspector.getTypeName());
+ throw new SerDeException(getClass().toString()
+ + " can only serialize struct types, but we got: "
+ + objInspector.getTypeName());
}
StructObjectInspector soi = (StructObjectInspector) objInspector;
List extends StructField> fields = soi.getAllStructFieldRefs();
-
+
StringBuilder sb = new StringBuilder();
- for(int i=0; i0) sb.append(separator);
+ for (int i = 0; i < fields.size(); i++) {
+ if (i > 0) {
+ sb.append(separator);
+ }
Object column = soi.getStructFieldData(obj, fields.get(i));
if (fields.get(i).getFieldObjectInspector().getCategory() == Category.PRIMITIVE) {
// For primitive object, serialize to plain string
sb.append(column == null ? nullString : column.toString());
} else {
// For complex object, serialize to JSON format
- sb.append(SerDeUtils.getJSONString(column, fields.get(i).getFieldObjectInspector()));
+ sb.append(SerDeUtils.getJSONString(column, fields.get(i)
+ .getFieldObjectInspector()));
}
}
serializeCache.set(sb.toString());
Index: serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryInteger.java
===================================================================
--- serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryInteger.java (revision 901519)
+++ serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryInteger.java (working copy)
@@ -24,9 +24,11 @@
/**
* LazyBinaryObject for integer which is serialized as VInt
- * @see LazyBinaryUtils#readVInt(byte[], int, VInt)
+ *
+ * @see LazyBinaryUtils#readVInt(byte[], int, VInt)
*/
-public class LazyBinaryInteger extends LazyBinaryPrimitive {
+public class LazyBinaryInteger extends
+ LazyBinaryPrimitive {
LazyBinaryInteger(WritableIntObjectInspector oi) {
super(oi);
@@ -39,14 +41,14 @@
}
/**
- * The reusable vInt for decoding the integer
+ * The reusable vInt for decoding the integer
*/
VInt vInt = new LazyBinaryUtils.VInt();
-
+
@Override
public void init(ByteArrayRef bytes, int start, int length) {
LazyBinaryUtils.readVInt(bytes.getData(), start, vInt);
- assert(length == vInt.length);
+ assert (length == vInt.length);
data.set(vInt.value);
}
}
Index: serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryLong.java
===================================================================
--- serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryLong.java (revision 901519)
+++ serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryLong.java (working copy)
@@ -24,29 +24,31 @@
/**
* LazyBinaryObject for long which stores as VLong.
+ *
* @see LazyBinaryUtils#readVLong(byte[], int, VLong)
*/
-public class LazyBinaryLong extends LazyBinaryPrimitive {
+public class LazyBinaryLong extends
+ LazyBinaryPrimitive {
LazyBinaryLong(WritableLongObjectInspector oi) {
super(oi);
data = new LongWritable();
}
-
+
LazyBinaryLong(LazyBinaryLong copy) {
super(copy);
data = new LongWritable(copy.data.get());
}
/**
- * The reusable vLong for decoding the long
+ * The reusable vLong for decoding the long
*/
VLong vLong = new LazyBinaryUtils.VLong();
-
+
@Override
public void init(ByteArrayRef bytes, int start, int length) {
LazyBinaryUtils.readVLong(bytes.getData(), start, vLong);
- assert(length == vLong.length);
+ assert (length == vLong.length);
data.set(vLong.value);
}
}
Index: serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryShort.java
===================================================================
--- serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryShort.java (revision 901519)
+++ serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryShort.java (working copy)
@@ -24,7 +24,8 @@
/**
* LazyBinaryObject for short which takes two bytes.
*/
-public class LazyBinaryShort extends LazyBinaryPrimitive {
+public class LazyBinaryShort extends
+ LazyBinaryPrimitive {
LazyBinaryShort(WritableShortObjectInspector oi) {
super(oi);
@@ -38,7 +39,7 @@
@Override
public void init(ByteArrayRef bytes, int start, int length) {
- assert(2 == length);
- data.set(LazyBinaryUtils.byteArrayToShort(bytes.getData(), start));
- }
+ assert (2 == length);
+ data.set(LazyBinaryUtils.byteArrayToShort(bytes.getData(), start));
+ }
}
Index: serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryUtils.java
===================================================================
--- serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryUtils.java (revision 901519)
+++ serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryUtils.java (working copy)
@@ -23,8 +23,8 @@
import org.apache.hadoop.hive.serde2.ByteStream.Output;
import org.apache.hadoop.hive.serde2.lazybinary.objectinspector.LazyBinaryObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category;
import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
@@ -36,206 +36,230 @@
import org.apache.hadoop.io.WritableUtils;
public class LazyBinaryUtils {
-
+
/**
- * Convert the byte array to an int starting from the given offset.
- * Refer to code by aeden on DZone Snippets:
- * @param b the byte array
- * @param offset the array offset
- * @return the integer
+ * Convert the byte array to an int starting from the given offset. Refer to
+ * code by aeden on DZone Snippets:
+ *
+ * @param b
+ * the byte array
+ * @param offset
+ * the array offset
+ * @return the integer
*/
public static int byteArrayToInt(byte[] b, int offset) {
- int value = 0;
- for (int i = 0; i < 4; i++) {
- int shift = (4 - 1 - i) * 8;
- value += (b[i + offset] & 0x000000FF) << shift;
- }
- return value;
+ int value = 0;
+ for (int i = 0; i < 4; i++) {
+ int shift = (4 - 1 - i) * 8;
+ value += (b[i + offset] & 0x000000FF) << shift;
+ }
+ return value;
}
-
+
/**
* Convert the byte array to a long starting from the given offset.
- * @param b the byte array
- * @param offset the array offset
- * @return the long
+ *
+ * @param b
+ * the byte array
+ * @param offset
+ * the array offset
+ * @return the long
*/
public static long byteArrayToLong(byte[] b, int offset) {
- long value = 0;
- for (int i = 0; i < 8; i++) {
- int shift = (8 - 1 - i) * 8;
- value += ((long) (b[i + offset] & 0x00000000000000FF)) << shift;
- }
- return value;
+ long value = 0;
+ for (int i = 0; i < 8; i++) {
+ int shift = (8 - 1 - i) * 8;
+ value += ((long) (b[i + offset] & 0x00000000000000FF)) << shift;
+ }
+ return value;
}
/**
* Convert the byte array to a short starting from the given offset.
- * @param b the byte array
- * @param offset the array offset
- * @return the short
+ *
+ * @param b
+ * the byte array
+ * @param offset
+ * the array offset
+ * @return the short
*/
public static short byteArrayToShort(byte[] b, int offset) {
- short value = 0;
- value += (b[offset ] & 0x000000FF) << 8;
- value += (b[offset+1] & 0x000000FF);
- return value;
+ short value = 0;
+ value += (b[offset] & 0x000000FF) << 8;
+ value += (b[offset + 1] & 0x000000FF);
+ return value;
}
-
+
/**
- * Record is the unit that data is serialized in.
- * A record includes two parts. The first part stores the
- * size of the element and the second part stores the
- * real element.
- * size element
- * record -> |----|-------------------------|
- *
- * A RecordInfo stores two information of a record,
- * the size of the "size" part which is the element offset
- * and the size of the element part which is element size.
- */
+ * Record is the unit that data is serialized in. A record includes two parts.
+ * The first part stores the size of the element and the second part stores
+ * the real element. size element record -> |----|-------------------------|
+ *
+ * A RecordInfo stores two information of a record, the size of the "size"
+ * part which is the element offset and the size of the element part which is
+ * element size.
+ */
public static class RecordInfo {
- public RecordInfo () {
+ public RecordInfo() {
elementOffset = 0;
- elementSize = 0;
+ elementSize = 0;
}
+
public byte elementOffset;
- public int elementSize;
+ public int elementSize;
}
static VInt vInt = new LazyBinaryUtils.VInt();
+
/**
- * Check a particular field and set its size and offset in bytes
- * based on the field type and the bytes arrays.
- *
- * For void, boolean, byte, short, int, long, float and double,
- * there is no offset and the size is fixed. For string, map,
- * list, struct, the first four bytes are used to store the size.
- * So the offset is 4 and the size is computed by concating the
- * first four bytes together. The first four bytes are defined
- * with respect to the offset in the bytes arrays.
+ * Check a particular field and set its size and offset in bytes based on the
+ * field type and the bytes arrays.
*
- * @param objectInspector object inspector of the field
- * @param bytes bytes arrays store the table row
- * @param offset offset of this field
- * @param recordInfo modify this byteinfo object and return it
+ * For void, boolean, byte, short, int, long, float and double, there is no
+ * offset and the size is fixed. For string, map, list, struct, the first four
+ * bytes are used to store the size. So the offset is 4 and the size is
+ * computed by concating the first four bytes together. The first four bytes
+ * are defined with respect to the offset in the bytes arrays.
+ *
+ * @param objectInspector
+ * object inspector of the field
+ * @param bytes
+ * bytes arrays store the table row
+ * @param offset
+ * offset of this field
+ * @param recordInfo
+ * modify this byteinfo object and return it
*/
- public static void checkObjectByteInfo(ObjectInspector objectInspector, byte[] bytes, int offset, RecordInfo recordInfo) {
- Category category = objectInspector.getCategory();
+ public static void checkObjectByteInfo(ObjectInspector objectInspector,
+ byte[] bytes, int offset, RecordInfo recordInfo) {
+ Category category = objectInspector.getCategory();
switch (category) {
case PRIMITIVE:
- PrimitiveCategory primitiveCategory = ((PrimitiveObjectInspector)objectInspector).getPrimitiveCategory();
+ PrimitiveCategory primitiveCategory = ((PrimitiveObjectInspector) objectInspector)
+ .getPrimitiveCategory();
switch (primitiveCategory) {
case VOID:
case BOOLEAN:
case BYTE:
recordInfo.elementOffset = 0;
- recordInfo.elementSize = 1;
+ recordInfo.elementSize = 1;
break;
case SHORT:
recordInfo.elementOffset = 0;
- recordInfo.elementSize = 2;
+ recordInfo.elementSize = 2;
break;
case FLOAT:
recordInfo.elementOffset = 0;
- recordInfo.elementSize = 4;
- break;
+ recordInfo.elementSize = 4;
+ break;
case DOUBLE:
recordInfo.elementOffset = 0;
- recordInfo.elementSize = 8;
- break;
+ recordInfo.elementSize = 8;
+ break;
case INT:
recordInfo.elementOffset = 0;
- recordInfo.elementSize = WritableUtils.decodeVIntSize(bytes[offset]);
+ recordInfo.elementSize = WritableUtils.decodeVIntSize(bytes[offset]);
break;
case LONG:
recordInfo.elementOffset = 0;
- recordInfo.elementSize = WritableUtils.decodeVIntSize(bytes[offset]);
+ recordInfo.elementSize = WritableUtils.decodeVIntSize(bytes[offset]);
break;
case STRING:
// using vint instead of 4 bytes
LazyBinaryUtils.readVInt(bytes, offset, vInt);
recordInfo.elementOffset = vInt.length;
- recordInfo.elementSize = vInt.value;
+ recordInfo.elementSize = vInt.value;
break;
- default: {
- throw new RuntimeException("Unrecognized primitive type: " + primitiveCategory);
- }
+ default: {
+ throw new RuntimeException("Unrecognized primitive type: "
+ + primitiveCategory);
}
+ }
break;
case LIST:
case MAP:
case STRUCT:
recordInfo.elementOffset = 4;
- recordInfo.elementSize = LazyBinaryUtils.byteArrayToInt(bytes, offset);
+ recordInfo.elementSize = LazyBinaryUtils.byteArrayToInt(bytes, offset);
break;
- default : {
- throw new RuntimeException("Unrecognized non-primitive type: " + category);
- }
+ default: {
+ throw new RuntimeException("Unrecognized non-primitive type: " + category);
}
+ }
}
-
+
/**
- * A zero-compressed encoded long
+ * A zero-compressed encoded long
*/
public static class VLong {
public VLong() {
- value = 0;
+ value = 0;
length = 0;
}
+
public long value;
- public byte length;
+ public byte length;
};
-
+
/**
* Reads a zero-compressed encoded long from a byte array and returns it.
- * @param bytes the byte array
- * @param offset offset of the array to read from
- * @param vlong storing the deserialized long and its size in byte
+ *
+ * @param bytes
+ * the byte array
+ * @param offset
+ * offset of the array to read from
+ * @param vlong
+ * storing the deserialized long and its size in byte
*/
public static void readVLong(byte[] bytes, int offset, VLong vlong) {
byte firstByte = bytes[offset];
- vlong.length = (byte)WritableUtils.decodeVIntSize(firstByte);
+ vlong.length = (byte) WritableUtils.decodeVIntSize(firstByte);
if (vlong.length == 1) {
vlong.value = firstByte;
return;
}
long i = 0;
- for (int idx = 0; idx < vlong.length-1; idx++) {
- byte b = bytes[offset+1+idx];
+ for (int idx = 0; idx < vlong.length - 1; idx++) {
+ byte b = bytes[offset + 1 + idx];
i = i << 8;
i = i | (b & 0xFF);
}
vlong.value = (WritableUtils.isNegativeVInt(firstByte) ? (i ^ -1L) : i);
}
-
+
/**
- * A zero-compressed encoded integer
+ * A zero-compressed encoded integer
*/
public static class VInt {
public VInt() {
- value = 0;
+ value = 0;
length = 0;
}
- public int value;
- public byte length;
+
+ public int value;
+ public byte length;
};
/**
* Reads a zero-compressed encoded int from a byte array and returns it.
- * @param bytes the byte array
- * @param offset offset of the array to read from
- * @param vInt storing the deserialized int and its size in byte
+ *
+ * @param bytes
+ * the byte array
+ * @param offset
+ * offset of the array to read from
+ * @param vInt
+ * storing the deserialized int and its size in byte
*/
public static void readVInt(byte[] bytes, int offset, VInt vInt) {
byte firstByte = bytes[offset];
- vInt.length = (byte)WritableUtils.decodeVIntSize(firstByte);
+ vInt.length = (byte) WritableUtils.decodeVIntSize(firstByte);
if (vInt.length == 1) {
vInt.value = firstByte;
return;
}
int i = 0;
- for (int idx = 0; idx < vInt.length-1; idx++) {
- byte b = bytes[offset+1+idx];
+ for (int idx = 0; idx < vInt.length - 1; idx++) {
+ byte b = bytes[offset + 1 + idx];
i = i << 8;
i = i | (b & 0xFF);
}
@@ -244,92 +268,111 @@
/**
* Writes a zero-compressed encoded int to a byte array.
- * @param byteStream the byte array/stream
- * @param i the int
+ *
+ * @param byteStream
+ * the byte array/stream
+ * @param i
+ * the int
*/
public static void writeVInt(Output byteStream, int i) {
writeVLong(byteStream, i);
}
-
+
/**
* Write a zero-compressed encoded long to a byte array.
- * @param byteStream the byte array/stream
- * @param l the long
+ *
+ * @param byteStream
+ * the byte array/stream
+ * @param l
+ * the long
*/
public static void writeVLong(Output byteStream, long l) {
if (l >= -112 && l <= 127) {
- byteStream.write((byte)l);
+ byteStream.write((byte) l);
return;
}
-
+
int len = -112;
if (l < 0) {
l ^= -1L; // take one's complement'
len = -120;
}
-
+
long tmp = l;
while (tmp != 0) {
tmp = tmp >> 8;
len--;
}
-
- byteStream.write((byte)len);
-
+
+ byteStream.write((byte) len);
+
len = (len < -120) ? -(len + 120) : -(len + 112);
-
+
for (int idx = len; idx != 0; idx--) {
int shiftbits = (idx - 1) * 8;
long mask = 0xFFL << shiftbits;
- byteStream.write((byte)((l & mask) >> shiftbits));
+ byteStream.write((byte) ((l & mask) >> shiftbits));
}
- }
-
+ }
+
static HashMap cachedLazyBinaryObjectInspector = new HashMap();
/**
- * Returns the lazy binary object inspector that can be used to inspect an
+ * Returns the lazy binary object inspector that can be used to inspect an
* lazy binary object of that typeInfo
*
- * For primitive types, we use the standard writable object inspector.
+ * For primitive types, we use the standard writable object inspector.
*/
- public static ObjectInspector getLazyBinaryObjectInspectorFromTypeInfo(TypeInfo typeInfo) {
+ public static ObjectInspector getLazyBinaryObjectInspectorFromTypeInfo(
+ TypeInfo typeInfo) {
ObjectInspector result = cachedLazyBinaryObjectInspector.get(typeInfo);
if (result == null) {
- switch(typeInfo.getCategory()) {
- case PRIMITIVE: {
- result = PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(
- ((PrimitiveTypeInfo)typeInfo).getPrimitiveCategory());
- break;
+ switch (typeInfo.getCategory()) {
+ case PRIMITIVE: {
+ result = PrimitiveObjectInspectorFactory
+ .getPrimitiveWritableObjectInspector(((PrimitiveTypeInfo) typeInfo)
+ .getPrimitiveCategory());
+ break;
+ }
+ case LIST: {
+ ObjectInspector elementObjectInspector = getLazyBinaryObjectInspectorFromTypeInfo(((ListTypeInfo) typeInfo)
+ .getListElementTypeInfo());
+ result = LazyBinaryObjectInspectorFactory
+ .getLazyBinaryListObjectInspector(elementObjectInspector);
+ break;
+ }
+ case MAP: {
+ MapTypeInfo mapTypeInfo = (MapTypeInfo) typeInfo;
+ ObjectInspector keyObjectInspector = getLazyBinaryObjectInspectorFromTypeInfo(mapTypeInfo
+ .getMapKeyTypeInfo());
+ ObjectInspector valueObjectInspector = getLazyBinaryObjectInspectorFromTypeInfo(mapTypeInfo
+ .getMapValueTypeInfo());
+ result = LazyBinaryObjectInspectorFactory
+ .getLazyBinaryMapObjectInspector(keyObjectInspector,
+ valueObjectInspector);
+ break;
+ }
+ case STRUCT: {
+ StructTypeInfo structTypeInfo = (StructTypeInfo) typeInfo;
+ List fieldNames = structTypeInfo.getAllStructFieldNames();
+ List fieldTypeInfos = structTypeInfo
+ .getAllStructFieldTypeInfos();
+ List fieldObjectInspectors = new ArrayList(
+ fieldTypeInfos.size());
+ for (int i = 0; i < fieldTypeInfos.size(); i++) {
+ fieldObjectInspectors
+ .add(getLazyBinaryObjectInspectorFromTypeInfo(fieldTypeInfos
+ .get(i)));
}
- case LIST: {
- ObjectInspector elementObjectInspector = getLazyBinaryObjectInspectorFromTypeInfo(
- ((ListTypeInfo)typeInfo).getListElementTypeInfo());
- result = LazyBinaryObjectInspectorFactory.getLazyBinaryListObjectInspector(elementObjectInspector);
- break;
- }
- case MAP: {
- MapTypeInfo mapTypeInfo = (MapTypeInfo)typeInfo;
- ObjectInspector keyObjectInspector = getLazyBinaryObjectInspectorFromTypeInfo(mapTypeInfo.getMapKeyTypeInfo());
- ObjectInspector valueObjectInspector = getLazyBinaryObjectInspectorFromTypeInfo(mapTypeInfo.getMapValueTypeInfo());
- result = LazyBinaryObjectInspectorFactory.getLazyBinaryMapObjectInspector(keyObjectInspector, valueObjectInspector);
- break;
- }
- case STRUCT: {
- StructTypeInfo structTypeInfo = (StructTypeInfo)typeInfo;
- List fieldNames = structTypeInfo.getAllStructFieldNames();
- List fieldTypeInfos = structTypeInfo.getAllStructFieldTypeInfos();
- List fieldObjectInspectors = new ArrayList(fieldTypeInfos.size());
- for(int i=0; i columnNames;
- List columnTypes;
-
- TypeInfo rowTypeInfo;
+ List columnTypes;
+
+ TypeInfo rowTypeInfo;
ObjectInspector cachedObjectInspector;
-
+
// The object for storing row data
LazyBinaryStruct cachedLazyBinaryStruct;
-
+
/**
* Initialize the SerDe with configuration and table information
*/
@@ -95,39 +95,41 @@
if (columnTypeProperty.length() == 0) {
columnTypes = new ArrayList();
} else {
- columnTypes = TypeInfoUtils.getTypeInfosFromTypeString(columnTypeProperty);
+ columnTypes = TypeInfoUtils
+ .getTypeInfosFromTypeString(columnTypeProperty);
}
- assert(columnNames.size() == columnTypes.size());
+ assert (columnNames.size() == columnTypes.size());
// Create row related objects
- rowTypeInfo = TypeInfoFactory.getStructTypeInfo(columnNames, columnTypes);
+ rowTypeInfo = TypeInfoFactory.getStructTypeInfo(columnNames, columnTypes);
// Create the object inspector and the lazy binary struct object
- cachedObjectInspector = LazyBinaryUtils.getLazyBinaryObjectInspectorFromTypeInfo(rowTypeInfo);
+ cachedObjectInspector = LazyBinaryUtils
+ .getLazyBinaryObjectInspectorFromTypeInfo(rowTypeInfo);
cachedLazyBinaryStruct = (LazyBinaryStruct) LazyBinaryFactory
- .createLazyBinaryObject(cachedObjectInspector);
+ .createLazyBinaryObject(cachedObjectInspector);
// output debug info
- LOG.debug("LazyBinarySerDe initialized with: columnNames="
- + columnNames + " columnTypes=" + columnTypes);
+ LOG.debug("LazyBinarySerDe initialized with: columnNames=" + columnNames
+ + " columnTypes=" + columnTypes);
}
-
+
/**
* Returns the ObjectInspector for the row.
*/
@Override
public ObjectInspector getObjectInspector() throws SerDeException {
- return cachedObjectInspector;
+ return cachedObjectInspector;
}
/**
* Returns the Writable Class after serialization.
*/
@Override
- public Class extends Writable> getSerializedClass() {
+ public Class extends Writable> getSerializedClass() {
return BytesWritable.class;
}
// The wrapper for byte array
ByteArrayRef byteArrayRef;
-
+
/**
* Deserialize a table record to a lazybinary struct.
*/
@@ -137,29 +139,31 @@
byteArrayRef = new ByteArrayRef();
}
if (field instanceof BytesWritable) {
- BytesWritable b = (BytesWritable)field;
- if(b.getSize()==0)
+ BytesWritable b = (BytesWritable) field;
+ if (b.getSize() == 0) {
return null;
+ }
// For backward-compatibility with hadoop 0.17
byteArrayRef.setData(b.get());
cachedLazyBinaryStruct.init(byteArrayRef, 0, b.getSize());
} else if (field instanceof Text) {
- Text t = (Text)field;
- if(t.getLength()==0)
+ Text t = (Text) field;
+ if (t.getLength() == 0) {
return null;
+ }
byteArrayRef.setData(t.getBytes());
cachedLazyBinaryStruct.init(byteArrayRef, 0, t.getLength());
} else {
- throw new SerDeException(getClass().toString()
+ throw new SerDeException(getClass().toString()
+ ": expects either BytesWritable or Text object!");
}
return cachedLazyBinaryStruct;
}
-
+
/**
* The reusable output buffer and serialize byte buffer.
*/
- BytesWritable serializeBytesWritable = new BytesWritable();
+ BytesWritable serializeBytesWritable = new BytesWritable();
ByteStream.Output serializeByteStream = new ByteStream.Output();
/**
@@ -170,293 +174,300 @@
throws SerDeException {
// make sure it is a struct record
if (objInspector.getCategory() != Category.STRUCT) {
- throw new SerDeException(getClass().toString()
- + " can only serialize struct types, but we got: "
+ throw new SerDeException(getClass().toString()
+ + " can only serialize struct types, but we got: "
+ objInspector.getTypeName());
}
-
- serializeByteStream.reset();
+
+ serializeByteStream.reset();
// serialize the row as a struct
- serializeStruct(serializeByteStream, obj,
- (StructObjectInspector) objInspector);
- // return the serialized bytes
- serializeBytesWritable.set(serializeByteStream.getData(),
- 0, serializeByteStream.getCount());
+ serializeStruct(serializeByteStream, obj,
+ (StructObjectInspector) objInspector);
+ // return the serialized bytes
+ serializeBytesWritable.set(serializeByteStream.getData(), 0,
+ serializeByteStream.getCount());
return serializeBytesWritable;
}
boolean nullMapKey = false;
-
+
/**
- * Serialize a struct object without writing the byte size.
- * This function is shared by both row serialization and
- * struct serialization.
+ * Serialize a struct object without writing the byte size. This function is
+ * shared by both row serialization and struct serialization.
*
- * @param byteStream the byte stream storing the serialization data
- * @param obj the struct object to serialize
- * @param objInspector the struct object inspector
+ * @param byteStream
+ * the byte stream storing the serialization data
+ * @param obj
+ * the struct object to serialize
+ * @param objInspector
+ * the struct object inspector
*/
- private void serializeStruct(Output byteStream,
- Object obj, StructObjectInspector soi) {
+ private void serializeStruct(Output byteStream, Object obj,
+ StructObjectInspector soi) {
// do nothing for null struct
- if(null == obj)
+ if (null == obj) {
return;
- /*
- * Interleave serializing one null byte and 8 struct fields
- * in each round, in order to support data deserialization
- * with different table schemas
+ }
+ /*
+ * Interleave serializing one null byte and 8 struct fields in each round,
+ * in order to support data deserialization with different table schemas
*/
- List extends StructField> fields = soi.getAllStructFieldRefs();
- int size = fields.size();
+ List extends StructField> fields = soi.getAllStructFieldRefs();
+ int size = fields.size();
int lasti = 0;
byte nullByte = 0;
- for (int i=0; i> 8));
- byteStream.write((byte) (v));
- return;
- }
- case INT: {
- IntObjectInspector ioi = (IntObjectInspector)poi;
- int v = ioi.get(obj);
- LazyBinaryUtils.writeVInt(byteStream, v);
- return;
- }
- case LONG: {
- LongObjectInspector loi = (LongObjectInspector)poi;
- long v = loi.get(obj);
- LazyBinaryUtils.writeVLong(byteStream, v);
- return;
- }
- case FLOAT: {
- FloatObjectInspector foi = (FloatObjectInspector)poi;
- int v = Float.floatToIntBits(foi.get(obj));
- byteStream.write((byte) (v >> 24));
- byteStream.write((byte) (v >> 16));
- byteStream.write((byte) (v >> 8));
- byteStream.write((byte) (v));
- return;
- }
- case DOUBLE: {
- DoubleObjectInspector doi = (DoubleObjectInspector)poi;
- long v = Double.doubleToLongBits(doi.get(obj));
- byteStream.write((byte) (v >> 56));
- byteStream.write((byte) (v >> 48));
- byteStream.write((byte) (v >> 40));
- byteStream.write((byte) (v >> 32));
- byteStream.write((byte) (v >> 24));
- byteStream.write((byte) (v >> 16));
- byteStream.write((byte) (v >> 8));
- byteStream.write((byte) (v));
- return;
- }
- case STRING: {
- StringObjectInspector soi = (StringObjectInspector)poi;
- Text t = soi.getPrimitiveWritableObject(obj);
- /* write byte size of the string which is a vint */
- int length = t.getLength();
- LazyBinaryUtils.writeVInt(byteStream, length);
- /* write string itself */
- byte[] data = t.getBytes();
- byteStream.write(data, 0, length);
- return;
- }
- default: {
- throw new RuntimeException("Unrecognized type: " + poi.getPrimitiveCategory());
- }
- }
+ case PRIMITIVE: {
+ PrimitiveObjectInspector poi = (PrimitiveObjectInspector) objInspector;
+ switch (poi.getPrimitiveCategory()) {
+ case VOID: {
+ return;
}
- case LIST: {
- ListObjectInspector loi = (ListObjectInspector)objInspector;
- ObjectInspector eoi = loi.getListElementObjectInspector();
-
- // 1/ reserve spaces for the byte size of the list
- // which is a integer and takes four bytes
- int byteSizeStart = byteStream.getCount();
- byteStream.write((byte) 0);
- byteStream.write((byte) 0);
- byteStream.write((byte) 0);
- byteStream.write((byte) 0);
- int listStart = byteStream.getCount();
-
- // 2/ write the size of the list as a VInt
- int size = loi.getListLength(obj);
- LazyBinaryUtils.writeVInt(byteStream, size);
-
- // 3/ write the null bytes
- byte nullByte = 0;
- for (int eid = 0; eid < size; eid++) {
- // set the bit to 1 if an element is not null
- if (null != loi.getListElement(obj, eid)) {
- nullByte |= 1 << (eid%8);
- }
- // store the byte every eight elements or
- // if this is the last element
- if (7 == eid%8 || eid == size-1) {
- byteStream.write(nullByte);
- nullByte = 0;
- }
- }
-
- // 4/ write element by element from the list
- for (int eid = 0; eid < size; eid++) {
- serialize(byteStream, loi.getListElement(obj, eid), eoi);
- }
-
- // 5/ update the list byte size
- int listEnd = byteStream.getCount();
- int listSize = listEnd - listStart;
- byte [] bytes = byteStream.getData();
- bytes[byteSizeStart ] = (byte) (listSize >> 24);
- bytes[byteSizeStart + 1] = (byte) (listSize >> 16);
- bytes[byteSizeStart + 2] = (byte) (listSize >> 8);
- bytes[byteSizeStart + 3] = (byte) (listSize);
-
+ case BOOLEAN: {
+ boolean v = ((BooleanObjectInspector) poi).get(obj);
+ byteStream.write((byte) (v ? 1 : 0));
return;
- }
- case MAP: {
- MapObjectInspector moi = (MapObjectInspector)objInspector;
- ObjectInspector koi = moi.getMapKeyObjectInspector();
- ObjectInspector voi = moi.getMapValueObjectInspector();
- Map, ?> map = moi.getMap(obj);
-
- // 1/ reserve spaces for the byte size of the map
- // which is a integer and takes four bytes
- int byteSizeStart = byteStream.getCount();
- byteStream.write((byte) 0);
- byteStream.write((byte) 0);
- byteStream.write((byte) 0);
- byteStream.write((byte) 0);
- int mapStart = byteStream.getCount();
-
- // 2/ write the size of the map which is a VInt
- int size = map.size();
- LazyBinaryUtils.writeVInt(byteStream, size);
-
- // 3/ write the null bytes
- int b = 0;
- byte nullByte = 0;
- for (Map.Entry, ?> entry: map.entrySet()) {
- // set the bit to 1 if a key is not null
- if (null != entry.getKey()) {
- nullByte |= 1 << (b%8);
- } else if (!nullMapKey) {
- nullMapKey = true;
- LOG.warn("Null map key encountered! Ignoring similar problems.");
- }
- b++;
- // set the bit to 1 if a value is not null
- if (null != entry.getValue()) {
- nullByte |= 1 << (b%8);
- }
- b++;
- // write the byte to stream every 4 key-value pairs
- // or if this is the last key-value pair
- if (0 == b%8 || b == size*2) {
- byteStream.write(nullByte);
- nullByte = 0;
- }
- }
-
- // 4/ write key-value pairs one by one
- for(Map.Entry, ?> entry: map.entrySet()) {
- serialize(byteStream, entry.getKey(), koi);
- serialize(byteStream, entry.getValue(), voi);
- }
-
- // 5/ update the byte size of the map
- int mapEnd = byteStream.getCount();
- int mapSize = mapEnd - mapStart;
- byte [] bytes = byteStream.getData();
- bytes[byteSizeStart ] = (byte) (mapSize >> 24);
- bytes[byteSizeStart + 1] = (byte) (mapSize >> 16);
- bytes[byteSizeStart + 2] = (byte) (mapSize >> 8);
- bytes[byteSizeStart + 3] = (byte) (mapSize);
-
+ }
+ case BYTE: {
+ ByteObjectInspector boi = (ByteObjectInspector) poi;
+ byte v = boi.get(obj);
+ byteStream.write(v);
return;
}
- case STRUCT: {
- // 1/ reserve spaces for the byte size of the struct
- // which is a integer and takes four bytes
- int byteSizeStart = byteStream.getCount();
- byteStream.write((byte) 0);
- byteStream.write((byte) 0);
- byteStream.write((byte) 0);
- byteStream.write((byte) 0);
- int structStart = byteStream.getCount();
-
- // 2/ serialize the struct
- serializeStruct(byteStream, obj, (StructObjectInspector) objInspector);
-
- // 3/ update the byte size of the struct
- int structEnd = byteStream.getCount();
- int structSize = structEnd - structStart;
- byte [] bytes = byteStream.getData();
- bytes[byteSizeStart ] = (byte) (structSize >> 24);
- bytes[byteSizeStart + 1] = (byte) (structSize >> 16);
- bytes[byteSizeStart + 2] = (byte) (structSize >> 8);
- bytes[byteSizeStart + 3] = (byte) (structSize);
-
+ case SHORT: {
+ ShortObjectInspector spoi = (ShortObjectInspector) poi;
+ short v = spoi.get(obj);
+ byteStream.write((byte) (v >> 8));
+ byteStream.write((byte) (v));
return;
}
+ case INT: {
+ IntObjectInspector ioi = (IntObjectInspector) poi;
+ int v = ioi.get(obj);
+ LazyBinaryUtils.writeVInt(byteStream, v);
+ return;
+ }
+ case LONG: {
+ LongObjectInspector loi = (LongObjectInspector) poi;
+ long v = loi.get(obj);
+ LazyBinaryUtils.writeVLong(byteStream, v);
+ return;
+ }
+ case FLOAT: {
+ FloatObjectInspector foi = (FloatObjectInspector) poi;
+ int v = Float.floatToIntBits(foi.get(obj));
+ byteStream.write((byte) (v >> 24));
+ byteStream.write((byte) (v >> 16));
+ byteStream.write((byte) (v >> 8));
+ byteStream.write((byte) (v));
+ return;
+ }
+ case DOUBLE: {
+ DoubleObjectInspector doi = (DoubleObjectInspector) poi;
+ long v = Double.doubleToLongBits(doi.get(obj));
+ byteStream.write((byte) (v >> 56));
+ byteStream.write((byte) (v >> 48));
+ byteStream.write((byte) (v >> 40));
+ byteStream.write((byte) (v >> 32));
+ byteStream.write((byte) (v >> 24));
+ byteStream.write((byte) (v >> 16));
+ byteStream.write((byte) (v >> 8));
+ byteStream.write((byte) (v));
+ return;
+ }
+ case STRING: {
+ StringObjectInspector soi = (StringObjectInspector) poi;
+ Text t = soi.getPrimitiveWritableObject(obj);
+ /* write byte size of the string which is a vint */
+ int length = t.getLength();
+ LazyBinaryUtils.writeVInt(byteStream, length);
+ /* write string itself */
+ byte[] data = t.getBytes();
+ byteStream.write(data, 0, length);
+ return;
+ }
default: {
- throw new RuntimeException("Unrecognized type: " + objInspector.getCategory());
+ throw new RuntimeException("Unrecognized type: "
+ + poi.getPrimitiveCategory());
}
+ }
}
+ case LIST: {
+ ListObjectInspector loi = (ListObjectInspector) objInspector;
+ ObjectInspector eoi = loi.getListElementObjectInspector();
+
+ // 1/ reserve spaces for the byte size of the list
+ // which is a integer and takes four bytes
+ int byteSizeStart = byteStream.getCount();
+ byteStream.write((byte) 0);
+ byteStream.write((byte) 0);
+ byteStream.write((byte) 0);
+ byteStream.write((byte) 0);
+ int listStart = byteStream.getCount();
+
+ // 2/ write the size of the list as a VInt
+ int size = loi.getListLength(obj);
+ LazyBinaryUtils.writeVInt(byteStream, size);
+
+ // 3/ write the null bytes
+ byte nullByte = 0;
+ for (int eid = 0; eid < size; eid++) {
+ // set the bit to 1 if an element is not null
+ if (null != loi.getListElement(obj, eid)) {
+ nullByte |= 1 << (eid % 8);
+ }
+ // store the byte every eight elements or
+ // if this is the last element
+ if (7 == eid % 8 || eid == size - 1) {
+ byteStream.write(nullByte);
+ nullByte = 0;
+ }
+ }
+
+ // 4/ write element by element from the list
+ for (int eid = 0; eid < size; eid++) {
+ serialize(byteStream, loi.getListElement(obj, eid), eoi);
+ }
+
+ // 5/ update the list byte size
+ int listEnd = byteStream.getCount();
+ int listSize = listEnd - listStart;
+ byte[] bytes = byteStream.getData();
+ bytes[byteSizeStart] = (byte) (listSize >> 24);
+ bytes[byteSizeStart + 1] = (byte) (listSize >> 16);
+ bytes[byteSizeStart + 2] = (byte) (listSize >> 8);
+ bytes[byteSizeStart + 3] = (byte) (listSize);
+
+ return;
+ }
+ case MAP: {
+ MapObjectInspector moi = (MapObjectInspector) objInspector;
+ ObjectInspector koi = moi.getMapKeyObjectInspector();
+ ObjectInspector voi = moi.getMapValueObjectInspector();
+ Map, ?> map = moi.getMap(obj);
+
+ // 1/ reserve spaces for the byte size of the map
+ // which is a integer and takes four bytes
+ int byteSizeStart = byteStream.getCount();
+ byteStream.write((byte) 0);
+ byteStream.write((byte) 0);
+ byteStream.write((byte) 0);
+ byteStream.write((byte) 0);
+ int mapStart = byteStream.getCount();
+
+ // 2/ write the size of the map which is a VInt
+ int size = map.size();
+ LazyBinaryUtils.writeVInt(byteStream, size);
+
+ // 3/ write the null bytes
+ int b = 0;
+ byte nullByte = 0;
+ for (Map.Entry, ?> entry : map.entrySet()) {
+ // set the bit to 1 if a key is not null
+ if (null != entry.getKey()) {
+ nullByte |= 1 << (b % 8);
+ } else if (!nullMapKey) {
+ nullMapKey = true;
+ LOG.warn("Null map key encountered! Ignoring similar problems.");
+ }
+ b++;
+ // set the bit to 1 if a value is not null
+ if (null != entry.getValue()) {
+ nullByte |= 1 << (b % 8);
+ }
+ b++;
+ // write the byte to stream every 4 key-value pairs
+ // or if this is the last key-value pair
+ if (0 == b % 8 || b == size * 2) {
+ byteStream.write(nullByte);
+ nullByte = 0;
+ }
+ }
+
+ // 4/ write key-value pairs one by one
+ for (Map.Entry, ?> entry : map.entrySet()) {
+ serialize(byteStream, entry.getKey(), koi);
+ serialize(byteStream, entry.getValue(), voi);
+ }
+
+ // 5/ update the byte size of the map
+ int mapEnd = byteStream.getCount();
+ int mapSize = mapEnd - mapStart;
+ byte[] bytes = byteStream.getData();
+ bytes[byteSizeStart] = (byte) (mapSize >> 24);
+ bytes[byteSizeStart + 1] = (byte) (mapSize >> 16);
+ bytes[byteSizeStart + 2] = (byte) (mapSize >> 8);
+ bytes[byteSizeStart + 3] = (byte) (mapSize);
+
+ return;
+ }
+ case STRUCT: {
+ // 1/ reserve spaces for the byte size of the struct
+ // which is a integer and takes four bytes
+ int byteSizeStart = byteStream.getCount();
+ byteStream.write((byte) 0);
+ byteStream.write((byte) 0);
+ byteStream.write((byte) 0);
+ byteStream.write((byte) 0);
+ int structStart = byteStream.getCount();
+
+ // 2/ serialize the struct
+ serializeStruct(byteStream, obj, (StructObjectInspector) objInspector);
+
+ // 3/ update the byte size of the struct
+ int structEnd = byteStream.getCount();
+ int structSize = structEnd - structStart;
+ byte[] bytes = byteStream.getData();
+ bytes[byteSizeStart] = (byte) (structSize >> 24);
+ bytes[byteSizeStart + 1] = (byte) (structSize >> 16);
+ bytes[byteSizeStart + 2] = (byte) (structSize >> 8);
+ bytes[byteSizeStart + 3] = (byte) (structSize);
+
+ return;
+ }
+ default: {
+ throw new RuntimeException("Unrecognized type: "
+ + objInspector.getCategory());
+ }
+ }
}
}
Index: serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryByte.java
===================================================================
--- serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryByte.java (revision 901519)
+++ serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryByte.java (working copy)
@@ -17,14 +17,15 @@
*/
package org.apache.hadoop.hive.serde2.lazybinary;
+import org.apache.hadoop.hive.serde2.io.ByteWritable;
import org.apache.hadoop.hive.serde2.lazy.ByteArrayRef;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableByteObjectInspector;
-import org.apache.hadoop.hive.serde2.io.ByteWritable;
/**
* LazyBinaryObject for byte which takes one byte
*/
-public class LazyBinaryByte extends LazyBinaryPrimitive {
+public class LazyBinaryByte extends
+ LazyBinaryPrimitive {
LazyBinaryByte(WritableByteObjectInspector oi) {
super(oi);
@@ -35,10 +36,10 @@
super(copy);
data = new ByteWritable(copy.data.get());
}
-
+
@Override
public void init(ByteArrayRef bytes, int start, int length) {
- assert(1 == length);
+ assert (1 == length);
data.set(bytes.getData()[start]);
}
}
Index: serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryFloat.java
===================================================================
--- serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryFloat.java (revision 901519)
+++ serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryFloat.java (working copy)
@@ -24,13 +24,14 @@
/**
* LazyBinaryObject for float which takes four bytes.
*/
-public class LazyBinaryFloat extends LazyBinaryPrimitive {
+public class LazyBinaryFloat extends
+ LazyBinaryPrimitive {
LazyBinaryFloat(WritableFloatObjectInspector oi) {
super(oi);
data = new FloatWritable();
}
-
+
LazyBinaryFloat(LazyBinaryFloat copy) {
super(copy);
data = new FloatWritable(copy.data.get());
@@ -39,6 +40,7 @@
@Override
public void init(ByteArrayRef bytes, int start, int length) {
assert (4 == length);
- data.set(Float.intBitsToFloat(LazyBinaryUtils.byteArrayToInt(bytes.getData(), start)));
+ data.set(Float.intBitsToFloat(LazyBinaryUtils.byteArrayToInt(bytes
+ .getData(), start)));
}
}
Index: serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryObject.java
===================================================================
--- serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryObject.java (revision 901519)
+++ serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryObject.java (working copy)
@@ -21,46 +21,51 @@
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
/**
- * LazyBinaryObject stores an object in a binary format in a byte[].
- * For example, a double takes four bytes.
+ * LazyBinaryObject stores an object in a binary format in a byte[]. For
+ * example, a double takes four bytes.
*
* A LazyBinaryObject can represent any primitive object or hierarchical object
* like string, list, map or struct.
*/
public abstract class LazyBinaryObject {
-
+
OI oi;
-
+
/**
* Create a LazyBinaryObject.
- * @param oi Derived classes can access meta information about this Lazy
- * Binary Object (e.g, length, null-bits) from it.
+ *
+ * @param oi
+ * Derived classes can access meta information about this Lazy Binary
+ * Object (e.g, length, null-bits) from it.
*/
protected LazyBinaryObject(OI oi) {
this.oi = oi;
}
-
+
/**
- * Set the data for this LazyBinaryObject.
- * We take ByteArrayRef instead of byte[] so that we will be able to drop
- * the reference to byte[] by a single assignment.
- * The ByteArrayRef object can be reused across multiple rows.
+ * Set the data for this LazyBinaryObject. We take ByteArrayRef instead of
+ * byte[] so that we will be able to drop the reference to byte[] by a single
+ * assignment. The ByteArrayRef object can be reused across multiple rows.
*
* Never call this function if the object represent a null!!!
*
- * @param bytes The wrapper of the byte[].
- * @param start The start position inside the bytes.
- * @param length The length of the data, starting from "start"
+ * @param bytes
+ * The wrapper of the byte[].
+ * @param start
+ * The start position inside the bytes.
+ * @param length
+ * The length of the data, starting from "start"
* @see ByteArrayRef
*/
public abstract void init(ByteArrayRef bytes, int start, int length);
-
+
/**
- * If the LazyBinaryObject is a primitive Object, then deserialize it and return
- * the actual primitive Object.
- * Otherwise (string, list, map, struct), return this.
+ * If the LazyBinaryObject is a primitive Object, then deserialize it and
+ * return the actual primitive Object. Otherwise (string, list, map, struct),
+ * return this.
*/
public abstract Object getObject();
-
+
+ @Override
public abstract int hashCode();
}
Index: serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryString.java
===================================================================
--- serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryString.java (revision 901519)
+++ serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryString.java (working copy)
@@ -17,34 +17,34 @@
*/
package org.apache.hadoop.hive.serde2.lazybinary;
-import org.apache.hadoop.io.Text;
import org.apache.hadoop.hive.serde2.lazy.ByteArrayRef;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableStringObjectInspector;
+import org.apache.hadoop.io.Text;
/**
- * The serialization of LazyBinaryString is very simple:
- * start A end
- * bytes[] -> |---------------------------------|
+ * The serialization of LazyBinaryString is very simple: start A end bytes[] ->
+ * |---------------------------------|
*
- * Section A is just an array of bytes which are exactly
- * the Text contained in this object.
+ * Section A is just an array of bytes which are exactly the Text contained in
+ * this object.
*
*/
-public class LazyBinaryString extends LazyBinaryPrimitive {
+public class LazyBinaryString extends
+ LazyBinaryPrimitive {
LazyBinaryString(WritableStringObjectInspector OI) {
super(OI);
data = new Text();
}
-
+
public LazyBinaryString(LazyBinaryString copy) {
super(copy);
data = new Text(copy.data);
- }
+ }
@Override
public void init(ByteArrayRef bytes, int start, int length) {
- assert(length > -1);
+ assert (length > -1);
data.set(bytes.getData(), start, length);
}
}
Index: serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryFactory.java
===================================================================
--- serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryFactory.java (revision 901519)
+++ serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryFactory.java (working copy)
@@ -35,39 +35,40 @@
public class LazyBinaryFactory {
/**
- * Create a lazy binary primitive class given the type name.
+ * Create a lazy binary primitive class given the type name.
*/
- public static LazyBinaryPrimitive,?> createLazyBinaryPrimitiveClass(PrimitiveObjectInspector oi) {
+ public static LazyBinaryPrimitive, ?> createLazyBinaryPrimitiveClass(
+ PrimitiveObjectInspector oi) {
PrimitiveCategory p = oi.getPrimitiveCategory();
- switch(p) {
- case BOOLEAN: {
- return new LazyBinaryBoolean((WritableBooleanObjectInspector)oi);
- }
- case BYTE: {
- return new LazyBinaryByte((WritableByteObjectInspector)oi);
- }
- case SHORT: {
- return new LazyBinaryShort((WritableShortObjectInspector)oi);
- }
- case INT: {
- return new LazyBinaryInteger((WritableIntObjectInspector)oi);
- }
- case LONG: {
- return new LazyBinaryLong((WritableLongObjectInspector)oi);
- }
- case FLOAT: {
- return new LazyBinaryFloat((WritableFloatObjectInspector)oi);
- }
- case DOUBLE: {
- return new LazyBinaryDouble((WritableDoubleObjectInspector)oi);
- }
- case STRING: {
- return new LazyBinaryString((WritableStringObjectInspector)oi);
- }
- default: {
- throw new RuntimeException("Internal error: no LazyBinaryObject for " + p);
- }
+ switch (p) {
+ case BOOLEAN: {
+ return new LazyBinaryBoolean((WritableBooleanObjectInspector) oi);
}
+ case BYTE: {
+ return new LazyBinaryByte((WritableByteObjectInspector) oi);
+ }
+ case SHORT: {
+ return new LazyBinaryShort((WritableShortObjectInspector) oi);
+ }
+ case INT: {
+ return new LazyBinaryInteger((WritableIntObjectInspector) oi);
+ }
+ case LONG: {
+ return new LazyBinaryLong((WritableLongObjectInspector) oi);
+ }
+ case FLOAT: {
+ return new LazyBinaryFloat((WritableFloatObjectInspector) oi);
+ }
+ case DOUBLE: {
+ return new LazyBinaryDouble((WritableDoubleObjectInspector) oi);
+ }
+ case STRING: {
+ return new LazyBinaryString((WritableStringObjectInspector) oi);
+ }
+ default: {
+ throw new RuntimeException("Internal error: no LazyBinaryObject for " + p);
+ }
+ }
}
/**
@@ -75,17 +76,17 @@
*/
public static LazyBinaryObject createLazyBinaryObject(ObjectInspector oi) {
ObjectInspector.Category c = oi.getCategory();
- switch(c) {
+ switch (c) {
case PRIMITIVE:
- return createLazyBinaryPrimitiveClass((PrimitiveObjectInspector)oi);
+ return createLazyBinaryPrimitiveClass((PrimitiveObjectInspector) oi);
case MAP:
- return new LazyBinaryMap((LazyBinaryMapObjectInspector)oi);
- case LIST:
- return new LazyBinaryArray((LazyBinaryListObjectInspector)oi);
+ return new LazyBinaryMap((LazyBinaryMapObjectInspector) oi);
+ case LIST:
+ return new LazyBinaryArray((LazyBinaryListObjectInspector) oi);
case STRUCT:
- return new LazyBinaryStruct((LazyBinaryStructObjectInspector)oi);
+ return new LazyBinaryStruct((LazyBinaryStructObjectInspector) oi);
}
throw new RuntimeException("Hive LazyBinarySerDe Internal error.");
- }
+ }
}
Index: serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/objectinspector/LazyBinaryMapObjectInspector.java
===================================================================
--- serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/objectinspector/LazyBinaryMapObjectInspector.java (revision 901519)
+++ serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/objectinspector/LazyBinaryMapObjectInspector.java (working copy)
@@ -25,6 +25,7 @@
/**
* ObjectInspector for LazyBinaryMap
+ *
* @see LazyBinaryMap
*/
public class LazyBinaryMapObjectInspector extends StandardMapObjectInspector {
@@ -39,7 +40,7 @@
if (data == null) {
return null;
}
- return ((LazyBinaryMap)data).getMap();
+ return ((LazyBinaryMap) data).getMap();
}
@Override
@@ -47,7 +48,7 @@
if (data == null) {
return -1;
}
- return ((LazyBinaryMap)data).getMapSize();
+ return ((LazyBinaryMap) data).getMapSize();
}
@Override
@@ -55,6 +56,6 @@
if (data == null) {
return -1;
}
- return ((LazyBinaryMap)data).getMapValueElement(key);
+ return ((LazyBinaryMap) data).getMapValueElement(key);
}
}
Index: serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/objectinspector/LazyBinaryStructObjectInspector.java
===================================================================
--- serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/objectinspector/LazyBinaryStructObjectInspector.java (revision 901519)
+++ serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/objectinspector/LazyBinaryStructObjectInspector.java (working copy)
@@ -26,29 +26,32 @@
/**
* ObjectInspector for LazyBinaryStruct
+ *
* @see LazyBinaryStruct
*/
-public class LazyBinaryStructObjectInspector extends StandardStructObjectInspector {
+public class LazyBinaryStructObjectInspector extends
+ StandardStructObjectInspector {
- protected LazyBinaryStructObjectInspector(List structFieldNames, List structFieldObjectInspectors) {
+ protected LazyBinaryStructObjectInspector(List structFieldNames,
+ List structFieldObjectInspectors) {
super(structFieldNames, structFieldObjectInspectors);
}
-
+
protected LazyBinaryStructObjectInspector(List fields) {
- super(fields);
+ super(fields);
}
-
+
@Override
public Object getStructFieldData(Object data, StructField fieldRef) {
if (data == null) {
return null;
}
- LazyBinaryStruct struct = (LazyBinaryStruct)data;
+ LazyBinaryStruct struct = (LazyBinaryStruct) data;
MyField f = (MyField) fieldRef;
-
+
int fieldID = f.getFieldID();
- assert(fieldID >= 0 && fieldID < fields.size());
-
+ assert (fieldID >= 0 && fieldID < fields.size());
+
return struct.getField(fieldID);
}
@@ -57,7 +60,7 @@
if (data == null) {
return null;
}
- LazyBinaryStruct struct = (LazyBinaryStruct)data;
+ LazyBinaryStruct struct = (LazyBinaryStruct) data;
return struct.getFieldsAsList();
}
}
Index: serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/objectinspector/LazyBinaryObjectInspectorFactory.java
===================================================================
--- serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/objectinspector/LazyBinaryObjectInspectorFactory.java (revision 901519)
+++ serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/objectinspector/LazyBinaryObjectInspectorFactory.java (working copy)
@@ -7,55 +7,62 @@
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
/**
- * ObjectInspectorFactory is the primary way to create new ObjectInspector instances.
+ * ObjectInspectorFactory is the primary way to create new ObjectInspector
+ * instances.
*
- * SerDe classes should call the static functions in this library to create an ObjectInspector
- * to return to the caller of SerDe2.getObjectInspector().
+ * SerDe classes should call the static functions in this library to create an
+ * ObjectInspector to return to the caller of SerDe2.getObjectInspector().
*
- * The reason of having caches here is that ObjectInspectors do not have an internal
- * state - so ObjectInspectors with the same construction parameters should
- * result in exactly the same ObjectInspector.
+ * The reason of having caches here is that ObjectInspectors do not have an
+ * internal state - so ObjectInspectors with the same construction parameters
+ * should result in exactly the same ObjectInspector.
*/
public class LazyBinaryObjectInspectorFactory {
- static HashMap, LazyBinaryStructObjectInspector> cachedLazyBinaryStructObjectInspector =
- new HashMap, LazyBinaryStructObjectInspector>();
- public static LazyBinaryStructObjectInspector getLazyBinaryStructObjectInspector(List structFieldNames,
+ static HashMap, LazyBinaryStructObjectInspector> cachedLazyBinaryStructObjectInspector = new HashMap, LazyBinaryStructObjectInspector>();
+
+ public static LazyBinaryStructObjectInspector getLazyBinaryStructObjectInspector(
+ List structFieldNames,
List structFieldObjectInspectors) {
ArrayList signature = new ArrayList();
signature.add(structFieldNames);
signature.add(structFieldObjectInspectors);
- LazyBinaryStructObjectInspector result = cachedLazyBinaryStructObjectInspector.get(signature);
+ LazyBinaryStructObjectInspector result = cachedLazyBinaryStructObjectInspector
+ .get(signature);
if (result == null) {
- result = new LazyBinaryStructObjectInspector(structFieldNames, structFieldObjectInspectors);
+ result = new LazyBinaryStructObjectInspector(structFieldNames,
+ structFieldObjectInspectors);
cachedLazyBinaryStructObjectInspector.put(signature, result);
}
return result;
}
-
- static HashMap, LazyBinaryListObjectInspector> cachedLazyBinaryListObjectInspector =
- new HashMap, LazyBinaryListObjectInspector>();
- public static LazyBinaryListObjectInspector getLazyBinaryListObjectInspector(
+
+ static HashMap, LazyBinaryListObjectInspector> cachedLazyBinaryListObjectInspector = new HashMap, LazyBinaryListObjectInspector>();
+
+ public static LazyBinaryListObjectInspector getLazyBinaryListObjectInspector(
ObjectInspector listElementObjectInspector) {
ArrayList signature = new ArrayList();
signature.add(listElementObjectInspector);
- LazyBinaryListObjectInspector result = cachedLazyBinaryListObjectInspector.get(signature);
+ LazyBinaryListObjectInspector result = cachedLazyBinaryListObjectInspector
+ .get(signature);
if (result == null) {
result = new LazyBinaryListObjectInspector(listElementObjectInspector);
cachedLazyBinaryListObjectInspector.put(signature, result);
}
return result;
}
-
- static HashMap, LazyBinaryMapObjectInspector> cachedLazyBinaryMapObjectInspector =
- new HashMap, LazyBinaryMapObjectInspector>();
- public static LazyBinaryMapObjectInspector getLazyBinaryMapObjectInspector(
- ObjectInspector mapKeyObjectInspector, ObjectInspector mapValueObjectInspector) {
+
+ static HashMap, LazyBinaryMapObjectInspector> cachedLazyBinaryMapObjectInspector = new HashMap, LazyBinaryMapObjectInspector>();
+
+ public static LazyBinaryMapObjectInspector getLazyBinaryMapObjectInspector(
+ ObjectInspector mapKeyObjectInspector,
+ ObjectInspector mapValueObjectInspector) {
ArrayList signature = new ArrayList();
signature.add(mapKeyObjectInspector);
signature.add(mapValueObjectInspector);
- LazyBinaryMapObjectInspector result = cachedLazyBinaryMapObjectInspector.get(signature);
+ LazyBinaryMapObjectInspector result = cachedLazyBinaryMapObjectInspector
+ .get(signature);
if (result == null) {
result = new LazyBinaryMapObjectInspector(mapKeyObjectInspector,
mapValueObjectInspector);
Index: serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/objectinspector/LazyBinaryListObjectInspector.java
===================================================================
--- serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/objectinspector/LazyBinaryListObjectInspector.java (revision 901519)
+++ serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/objectinspector/LazyBinaryListObjectInspector.java (working copy)
@@ -20,8 +20,8 @@
import java.util.List;
import org.apache.hadoop.hive.serde2.lazybinary.LazyBinaryArray;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.StandardListObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
/**
* ObjectInspector for LazyBinaryList
Index: serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryPrimitive.java
===================================================================
--- serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryPrimitive.java (revision 901519)
+++ serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryPrimitive.java (working copy)
@@ -21,32 +21,33 @@
import org.apache.hadoop.io.Writable;
/**
- * Defines a LazyBinaryPrimitive.
+ * Defines a LazyBinaryPrimitive.
*
- * data will be initialized to corresponding types in
- * different LazyBinary primitive classes. For example, data will
- * be a BooleanWritable in the LazyBinaryBoolean class.
+ * data will be initialized to corresponding types in different LazyBinary
+ * primitive classes. For example, data will be a BooleanWritable in the
+ * LazyBinaryBoolean class.
*
- * There is no null flag any more,
- *
+ * There is no null flag any more,
+ *
*/
-public abstract class LazyBinaryPrimitive extends LazyBinaryObject {
+public abstract class LazyBinaryPrimitive
+ extends LazyBinaryObject {
LazyBinaryPrimitive(OI oi) {
super(oi);
}
-
+
LazyBinaryPrimitive(LazyBinaryPrimitive copy) {
super(copy.oi);
}
T data;
-
+
/**
- * Returns the primitive object represented by this LazyBinaryObject.
- * This is useful because it can make sure we have "null" for null objects.
+ * Returns the primitive object represented by this LazyBinaryObject. This is
+ * useful because it can make sure we have "null" for null objects.
*/
+ @Override
public Object getObject() {
return data;
}
@@ -54,12 +55,14 @@
public T getWritableObject() {
return data;
}
-
+
+ @Override
public String toString() {
return data.toString();
}
-
- public int hashCode(){
+
+ @Override
+ public int hashCode() {
return data == null ? 0 : data.hashCode();
}
}
Index: serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryDouble.java
===================================================================
--- serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryDouble.java (revision 901519)
+++ serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryDouble.java (working copy)
@@ -17,21 +17,21 @@
*/
package org.apache.hadoop.hive.serde2.lazybinary;
-
+import org.apache.hadoop.hive.serde2.io.DoubleWritable;
import org.apache.hadoop.hive.serde2.lazy.ByteArrayRef;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableDoubleObjectInspector;
-import org.apache.hadoop.hive.serde2.io.DoubleWritable;
/**
* LazyBinaryObject for double which takes eight bytes
*/
-public class LazyBinaryDouble extends LazyBinaryPrimitive {
+public class LazyBinaryDouble extends
+ LazyBinaryPrimitive {
LazyBinaryDouble(WritableDoubleObjectInspector oi) {
super(oi);
data = new DoubleWritable();
}
-
+
LazyBinaryDouble(LazyBinaryDouble copy) {
super(copy);
data = new DoubleWritable(copy.data.get());
@@ -39,7 +39,8 @@
@Override
public void init(ByteArrayRef bytes, int start, int length) {
- assert(8 == length);
- data.set(Double.longBitsToDouble(LazyBinaryUtils.byteArrayToLong(bytes.getData(), start)));
+ assert (8 == length);
+ data.set(Double.longBitsToDouble(LazyBinaryUtils.byteArrayToLong(bytes
+ .getData(), start)));
}
}
Index: serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryMap.java
===================================================================
--- serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryMap.java (revision 901519)
+++ serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryMap.java (working copy)
@@ -24,8 +24,6 @@
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hive.serde2.lazy.ByteArrayRef;
-import org.apache.hadoop.hive.serde2.lazy.LazyObject;
-import org.apache.hadoop.hive.serde2.lazy.LazyPrimitive;
import org.apache.hadoop.hive.serde2.lazybinary.LazyBinaryUtils.RecordInfo;
import org.apache.hadoop.hive.serde2.lazybinary.LazyBinaryUtils.VInt;
import org.apache.hadoop.hive.serde2.lazybinary.objectinspector.LazyBinaryMapObjectInspector;
@@ -33,22 +31,22 @@
import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
/**
- * LazyBinaryMap is serialized as follows:
- * start A b c b c b c end
- * bytes[] -> |--------|---|---|---|---| ... |---|---|
+ * LazyBinaryMap is serialized as follows: start A b c b c b c end bytes[] ->
+ * |--------|---|---|---|---| ... |---|---|
*
- * Section A is the null-bytes. Suppose the map has
- * N key-value pairs, then there are (N*2+7)/8 bytes used as null-bytes.
- * Each bit corresponds to a key or a value and it indicates whether
- * that key or value is null (0) or not null (1).
+ * Section A is the null-bytes. Suppose the map has N key-value pairs, then
+ * there are (N*2+7)/8 bytes used as null-bytes. Each bit corresponds to a key
+ * or a value and it indicates whether that key or value is null (0) or not null
+ * (1).
*
- * After A, all the bytes are actual serialized data of the map,
- * which are key-value pairs. b represent the keys and c represent
- * the values. Each of them is again a LazyBinaryObject.
- *
+ * After A, all the bytes are actual serialized data of the map, which are
+ * key-value pairs. b represent the keys and c represent the values. Each of
+ * them is again a LazyBinaryObject.
+ *
*/
-public class LazyBinaryMap extends LazyBinaryNonPrimitive {
+public class LazyBinaryMap extends
+ LazyBinaryNonPrimitive {
private static Log LOG = LogFactory.getLog(LazyBinaryMap.class.getName());
@@ -56,17 +54,16 @@
* Whether the data is already parsed or not.
*/
boolean parsed;
-
+
/**
- * The size of the map.
- * Only valid when the data is parsed.
- * -1 when the map is NULL.
+ * The size of the map. Only valid when the data is parsed. -1 when the map is
+ * NULL.
*/
int mapSize = 0;
-
+
/**
- * The beginning position and length of key[i] and value[i].
- * Only valid when the data is parsed.
+ * The beginning position and length of key[i] and value[i]. Only valid when
+ * the data is parsed.
*/
int[] keyStart;
int[] keyLength;
@@ -79,30 +76,30 @@
boolean[] valueInited;
/**
- * Whether valueObjects[i]/keyObjects[i] is null or not
- * This could not be inferred from the length of the object.
- * In particular, a 0-length string is not null.
+ * Whether valueObjects[i]/keyObjects[i] is null or not This could not be
+ * inferred from the length of the object. In particular, a 0-length string is
+ * not null.
*/
boolean[] keyIsNull;
boolean[] valueIsNull;
-
+
/**
* The keys are stored in an array of LazyPrimitives.
*/
- LazyBinaryPrimitive,?>[] keyObjects;
+ LazyBinaryPrimitive, ?>[] keyObjects;
/**
- * The values are stored in an array of LazyObjects.
- * value[index] will start from KeyEnd[index] + 1,
- * and ends before KeyStart[index+1] - 1.
+ * The values are stored in an array of LazyObjects. value[index] will start
+ * from KeyEnd[index] + 1, and ends before KeyStart[index+1] - 1.
*/
LazyBinaryObject[] valueObjects;
-
+
protected LazyBinaryMap(LazyBinaryMapObjectInspector oi) {
super(oi);
}
-
+
/**
* Set the row data for this LazyBinaryMap.
+ *
* @see LazyBinaryObject#init(ByteArrayRef, int, int)
*/
@Override
@@ -112,66 +109,63 @@
}
/**
- * Adjust the size of arrays:
- * keyStart, keyLength
- * valueStart, valueLength
- * keyInited, keyIsNull
- * valueInited, valueIsNull
+ * Adjust the size of arrays: keyStart, keyLength valueStart, valueLength
+ * keyInited, keyIsNull valueInited, valueIsNull
*/
protected void adjustArraySize(int newSize) {
if (keyStart == null || keyStart.length < newSize) {
- keyStart = new int[newSize];
- keyLength = new int[newSize];
- valueStart = new int[newSize];
- valueLength = new int[newSize];
- keyInited = new boolean[newSize];
- keyIsNull = new boolean[newSize];
- valueInited = new boolean[newSize];
- valueIsNull = new boolean[newSize];
- keyObjects = new LazyBinaryPrimitive,?>[newSize];
+ keyStart = new int[newSize];
+ keyLength = new int[newSize];
+ valueStart = new int[newSize];
+ valueLength = new int[newSize];
+ keyInited = new boolean[newSize];
+ keyIsNull = new boolean[newSize];
+ valueInited = new boolean[newSize];
+ valueIsNull = new boolean[newSize];
+ keyObjects = new LazyBinaryPrimitive, ?>[newSize];
valueObjects = new LazyBinaryObject[newSize];
}
- }
-
+ }
+
boolean nullMapKey = false;
VInt vInt = new LazyBinaryUtils.VInt();
- RecordInfo recordInfo = new LazyBinaryUtils.RecordInfo();
+ RecordInfo recordInfo = new LazyBinaryUtils.RecordInfo();
/**
- * Parse the byte[] and fill keyStart, keyLength, keyIsNull
- * valueStart, valueLength and valueIsNull
+ * Parse the byte[] and fill keyStart, keyLength, keyIsNull valueStart,
+ * valueLength and valueIsNull
*/
- private void parse() {
-
+ private void parse() {
+
byte[] bytes = this.bytes.getData();
-
+
// get the VInt that represents the map size
- LazyBinaryUtils.readVInt(bytes, start, vInt);
+ LazyBinaryUtils.readVInt(bytes, start, vInt);
mapSize = vInt.value;
- if(0 == mapSize) {
+ if (0 == mapSize) {
parsed = true;
return;
}
-
+
// adjust arrays
adjustArraySize(mapSize);
// find out the null-bytes
- int mapByteStart = start + vInt.length;
- int nullByteCur = mapByteStart;
- int nullByteEnd = mapByteStart + (mapSize*2+7) / 8;
+ int mapByteStart = start + vInt.length;
+ int nullByteCur = mapByteStart;
+ int nullByteEnd = mapByteStart + (mapSize * 2 + 7) / 8;
int lastElementByteEnd = nullByteEnd;
// parsing the keys and values one by one
- for (int i=0; i lazyKeyI = uncheckedGetKey(i);
- if (lazyKeyI == null) continue;
- // getWritableObject() will convert LazyPrimitive to actual primitive writable objects.
+ for (int i = 0; i < mapSize; i++) {
+ LazyBinaryPrimitive, ?> lazyKeyI = uncheckedGetKey(i);
+ if (lazyKeyI == null) {
+ continue;
+ }
+ // getWritableObject() will convert LazyPrimitive to actual primitive
+ // writable objects.
Object keyI = lazyKeyI.getWritableObject();
- if (keyI == null) continue;
+ if (keyI == null) {
+ continue;
+ }
if (keyI.equals(key)) {
// Got a match, return the value
LazyBinaryObject v = uncheckedGetValue(i);
return v == null ? v : v.getObject();
}
- }
+ }
return null;
}
-
/**
* Get the key object with the index without checking parsed.
- * @param index The index into the array starting from 0
+ *
+ * @param index
+ * The index into the array starting from 0
*/
- private LazyBinaryPrimitive,?> uncheckedGetKey(int index) {
+ private LazyBinaryPrimitive, ?> uncheckedGetKey(int index) {
if (keyIsNull[index]) {
return null;
}
@@ -266,24 +271,25 @@
keyInited[index] = true;
if (keyObjects[index] == null) {
// Keys are always primitive
- keyObjects[index] = LazyBinaryFactory.createLazyBinaryPrimitiveClass(
- (PrimitiveObjectInspector)((MapObjectInspector)oi).getMapKeyObjectInspector());
+ keyObjects[index] = LazyBinaryFactory
+ .createLazyBinaryPrimitiveClass((PrimitiveObjectInspector) ((MapObjectInspector) oi)
+ .getMapKeyObjectInspector());
}
keyObjects[index].init(bytes, keyStart[index], keyLength[index]);
}
return keyObjects[index];
}
-
+
/**
- * cachedMap is reused for different calls to getMap().
- * But each LazyBinaryMap has a separate cachedMap so we won't overwrite the
- * data by accident.
+ * cachedMap is reused for different calls to getMap(). But each LazyBinaryMap
+ * has a separate cachedMap so we won't overwrite the data by accident.
*/
LinkedHashMap cachedMap;
-
+
/**
- * Return the map object representing this LazyBinaryMap.
- * Note that the keyObjects will be Writable primitive objects.
+ * Return the map object representing this LazyBinaryMap. Note that the
+ * keyObjects will be Writable primitive objects.
+ *
* @return the map object
*/
public Map getMap() {
@@ -296,11 +302,13 @@
} else {
cachedMap.clear();
}
-
+
// go through each element of the map
for (int i = 0; i < mapSize; i++) {
- LazyBinaryPrimitive,?> lazyKey = uncheckedGetKey(i);
- if (lazyKey == null) continue;
+ LazyBinaryPrimitive, ?> lazyKey = uncheckedGetKey(i);
+ if (lazyKey == null) {
+ continue;
+ }
Object key = lazyKey.getObject();
// do not overwrite if there are duplicate keys
if (key != null && !cachedMap.containsKey(key)) {
@@ -311,15 +319,16 @@
}
return cachedMap;
}
-
+
/**
* Get the size of the map represented by this LazyBinaryMap.
- * @return The size of the map
+ *
+ * @return The size of the map
*/
public int getMapSize() {
if (!parsed) {
parse();
}
return mapSize;
- }
+ }
}
\ No newline at end of file
Index: serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryArray.java
===================================================================
--- serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryArray.java (revision 901519)
+++ serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryArray.java (working copy)
@@ -30,68 +30,67 @@
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
/**
- * LazyBinaryArray is serialized as follows:
- * start A b b b b b b end
- * bytes[] -> |--------|---|---|---|---| ... |---|---|
+ * LazyBinaryArray is serialized as follows: start A b b b b b b end bytes[] ->
+ * |--------|---|---|---|---| ... |---|---|
*
- * Section A is the null-bytes. Suppose the list has
- * N elements, then there are (N+7)/8 bytes used as null-bytes.
- * Each bit corresponds to an element and it indicates whether
- * that element is null (0) or not null (1).
+ * Section A is the null-bytes. Suppose the list has N elements, then there are
+ * (N+7)/8 bytes used as null-bytes. Each bit corresponds to an element and it
+ * indicates whether that element is null (0) or not null (1).
*
- * After A, all b(s) represent the elements of the list.
- * Each of them is again a LazyBinaryObject.
- *
+ * After A, all b(s) represent the elements of the list. Each of them is again a
+ * LazyBinaryObject.
+ *
*/
-public class LazyBinaryArray extends LazyBinaryNonPrimitive {
-
+public class LazyBinaryArray extends
+ LazyBinaryNonPrimitive {
+
/**
* Whether the data is already parsed or not.
*/
boolean parsed = false;
/**
- * The length of the array.
- * Only valid when the data is parsed.
+ * The length of the array. Only valid when the data is parsed.
*/
int arraySize = 0;
-
+
/**
- * The start positions and lengths of array elements.
- * Only valid when the data is parsed.
+ * The start positions and lengths of array elements. Only valid when the data
+ * is parsed.
*/
int[] elementStart;
int[] elementLength;
-
+
/**
* Whether an element is initialized or not
*/
boolean[] elementInited;
/**
- * Whether an element is null or not.
- * Because length is 0 does not means the field is null.
- * In particular, a 0-length string is not null.
+ * Whether an element is null or not. Because length is 0 does not means the
+ * field is null. In particular, a 0-length string is not null.
*/
boolean[] elementIsNull;
/**
- * The elements of the array. Note that we call
- * arrayElements[i].init(bytes, begin, length)
- * only when that element is accessed.
+ * The elements of the array. Note that we call arrayElements[i].init(bytes,
+ * begin, length) only when that element is accessed.
*/
LazyBinaryObject[] arrayElements;
/**
* Construct a LazyBinaryArray object with the ObjectInspector.
- * @param oi the oi representing the type of this LazyBinaryArray
+ *
+ * @param oi
+ * the oi representing the type of this LazyBinaryArray
*/
protected LazyBinaryArray(LazyBinaryListObjectInspector oi) {
super(oi);
}
-
+
/**
* Set the row data for this LazyBinaryArray.
+ *
* @see LazyObject#init(ByteArrayRef, int, int)
*/
@Override
@@ -99,13 +98,13 @@
super.init(bytes, start, length);
parsed = false;
}
-
+
/**
- * Enlarge the size of arrays storing information for the elements inside
- * the array.
+ * Enlarge the size of arrays storing information for the elements inside the
+ * array.
*/
private void adjustArraySize(int newSize) {
- if (elementStart == null || elementStart.length < newSize) {
+ if (elementStart == null || elementStart.length < newSize) {
elementStart = new int[newSize];
elementLength = new int[newSize];
elementInited = new boolean[newSize];
@@ -113,124 +112,127 @@
arrayElements = new LazyBinaryObject[newSize];
}
}
-
+
VInt vInt = new LazyBinaryUtils.VInt();
- RecordInfo recordInfo = new LazyBinaryUtils.RecordInfo();
+ RecordInfo recordInfo = new LazyBinaryUtils.RecordInfo();
+
/**
- * Parse the bytes and fill elementStart, elementLength,
- * elementInited and elementIsNull.
+ * Parse the bytes and fill elementStart, elementLength, elementInited and
+ * elementIsNull.
*/
private void parse() {
-
+
byte[] bytes = this.bytes.getData();
-
+
// get the vlong that represents the map size
- LazyBinaryUtils.readVInt(bytes, start, vInt);
+ LazyBinaryUtils.readVInt(bytes, start, vInt);
arraySize = vInt.value;
- if(0 == arraySize) {
+ if (0 == arraySize) {
parsed = true;
return;
}
-
+
// adjust arrays
adjustArraySize(arraySize);
// find out the null-bytes
- int arryByteStart = start + vInt.length;
- int nullByteCur = arryByteStart;
- int nullByteEnd = arryByteStart + (arraySize+7) / 8;
+ int arryByteStart = start + vInt.length;
+ int nullByteCur = arryByteStart;
+ int nullByteEnd = arryByteStart + (arraySize + 7) / 8;
// the begin the real elements
int lastElementByteEnd = nullByteEnd;
// the list element object inspector
- ObjectInspector listEleObjectInspector =
- ((ListObjectInspector)oi).getListElementObjectInspector();
+ ObjectInspector listEleObjectInspector = ((ListObjectInspector) oi)
+ .getListElementObjectInspector();
// parsing elements one by one
- for (int i=0; i= arraySize) {
- return null;
- }
- return uncheckedGetElement(index);
- }
-
- /**
- * Get the element without checking out-of-bound index.
- * @param index index to the array element
- */
- private Object uncheckedGetElement(int index) {
-
- if (elementIsNull[index]) {
- return null;
- } else {
- if (!elementInited[index]) {
- elementInited[index] = true;
- if (arrayElements[index] == null) {
- arrayElements[index] = LazyBinaryFactory.createLazyBinaryObject(
- ((LazyBinaryListObjectInspector)oi).getListElementObjectInspector());
- }
- arrayElements[index].init(bytes, elementStart[index], elementLength[index]);
- }
- }
- return arrayElements[index].getObject();
- }
-
- /**
- * Returns the array size.
- */
- public int getListLength() {
- if (!parsed) {
- parse();
- }
- return arraySize;
- }
-
- /**
- * cachedList is reused every time getList is called.
- * Different LazyBianryArray instances cannot share
- * the same cachedList.
- */
- ArrayList cachedList;
-
- /** Returns the List of actual primitive objects.
- * Returns null for null array.
- */
- public List getList() {
- if (!parsed) {
- parse();
- }
- if (cachedList == null) {
- cachedList = new ArrayList(arraySize);
- } else {
- cachedList.clear();
- }
- for (int index=0; index= arraySize) {
+ return null;
+ }
+ return uncheckedGetElement(index);
+ }
+
+ /**
+ * Get the element without checking out-of-bound index.
+ *
+ * @param index
+ * index to the array element
+ */
+ private Object uncheckedGetElement(int index) {
+
+ if (elementIsNull[index]) {
+ return null;
+ } else {
+ if (!elementInited[index]) {
+ elementInited[index] = true;
+ if (arrayElements[index] == null) {
+ arrayElements[index] = LazyBinaryFactory.createLazyBinaryObject((oi)
+ .getListElementObjectInspector());
+ }
+ arrayElements[index].init(bytes, elementStart[index],
+ elementLength[index]);
+ }
+ }
+ return arrayElements[index].getObject();
+ }
+
+ /**
+ * Returns the array size.
+ */
+ public int getListLength() {
+ if (!parsed) {
+ parse();
+ }
+ return arraySize;
+ }
+
+ /**
+ * cachedList is reused every time getList is called. Different
+ * LazyBianryArray instances cannot share the same cachedList.
+ */
+ ArrayList cachedList;
+
+ /**
+ * Returns the List of actual primitive objects. Returns null for null array.
+ */
+ public List getList() {
+ if (!parsed) {
+ parse();
+ }
+ if (cachedList == null) {
+ cachedList = new ArrayList(arraySize);
+ } else {
+ cachedList.clear();
+ }
+ for (int index = 0; index < arraySize; index++) {
+ cachedList.add(uncheckedGetElement(index));
+ }
+ return cachedList;
+ }
}
Index: serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryBoolean.java
===================================================================
--- serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryBoolean.java (revision 901519)
+++ serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryBoolean.java (working copy)
@@ -34,7 +34,8 @@
*
*
*/
-public class LazyBinaryBoolean extends LazyBinaryPrimitive {
+public class LazyBinaryBoolean extends
+ LazyBinaryPrimitive {
public LazyBinaryBoolean(WritableBooleanObjectInspector oi) {
super(oi);
@@ -48,12 +49,12 @@
@Override
public void init(ByteArrayRef bytes, int start, int length) {
- assert(1 == length);
+ assert (1 == length);
byte val = bytes.getData()[start];
if (val == 0) {
- data.set(false);
+ data.set(false);
} else if (val == 1) {
- data.set(true);
+ data.set(true);
}
}
}
Index: serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryNonPrimitive.java
===================================================================
--- serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryNonPrimitive.java (revision 901519)
+++ serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryNonPrimitive.java (working copy)
@@ -21,13 +21,14 @@
import org.apache.hadoop.hive.serde2.lazy.LazyUtils;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
-public abstract class LazyBinaryNonPrimitive extends LazyBinaryObject {
+public abstract class LazyBinaryNonPrimitive
+ extends LazyBinaryObject {
protected ByteArrayRef bytes;
protected int start;
protected int length;
- protected LazyBinaryNonPrimitive(OI oi) {
+ protected LazyBinaryNonPrimitive(OI oi) {
super(oi);
bytes = null;
start = 0;
@@ -51,7 +52,8 @@
this.start = start;
this.length = length;
}
-
+
+ @Override
public int hashCode() {
return LazyUtils.hashBytes(bytes.getData(), start, length);
}
Index: serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryStruct.java
===================================================================
--- serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryStruct.java (revision 901519)
+++ serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryStruct.java (working copy)
@@ -30,18 +30,18 @@
import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
/**
- * LazyBinaryStruct is serialized as follows:
- * start A B A B A B end
- * bytes[] -> |-----|---------|--- ... ---|-----|---------|
+ * LazyBinaryStruct is serialized as follows: start A B A B A B end bytes[] ->
+ * |-----|---------|--- ... ---|-----|---------|
*
- * Section A is one null-byte, corresponding to eight struct fields in Section B.
- * Each bit indicates whether the corresponding field is null (0) or not null (1).
- * Each field is a LazyBinaryObject.
+ * Section A is one null-byte, corresponding to eight struct fields in Section
+ * B. Each bit indicates whether the corresponding field is null (0) or not null
+ * (1). Each field is a LazyBinaryObject.
*
- * Following B, there is another section A and B. This pattern repeats until the
+ * Following B, there is another section A and B. This pattern repeats until the
* all struct fields are serialized.
*/
-public class LazyBinaryStruct extends LazyBinaryNonPrimitive {
+public class LazyBinaryStruct extends
+ LazyBinaryNonPrimitive {
private static Log LOG = LogFactory.getLog(LazyBinaryStruct.class.getName());
@@ -54,26 +54,25 @@
* The fields of the struct.
*/
LazyBinaryObject[] fields;
-
+
/**
* Whether a field is initialized or not.
*/
boolean[] fieldInited;
-
+
/**
- * Whether a field is null or not.
- * Because length is 0 does not means the field is null.
- * In particular, a 0-length string is not null.
+ * Whether a field is null or not. Because length is 0 does not means the
+ * field is null. In particular, a 0-length string is not null.
*/
boolean[] fieldIsNull;
-
+
/**
- * The start positions and lengths of struct fields.
- * Only valid when the data is parsed.
+ * The start positions and lengths of struct fields. Only valid when the data
+ * is parsed.
*/
- int[] fieldStart;
+ int[] fieldStart;
int[] fieldLength;
-
+
/**
* Construct a LazyBinaryStruct object with an ObjectInspector.
*/
@@ -84,77 +83,81 @@
@Override
public void init(ByteArrayRef bytes, int start, int length) {
super.init(bytes, start, length);
- parsed = false;
+ parsed = false;
}
-
- RecordInfo recordInfo = new LazyBinaryUtils.RecordInfo();
+
+ RecordInfo recordInfo = new LazyBinaryUtils.RecordInfo();
boolean missingFieldWarned = false;
boolean extraFieldWarned = false;
+
/**
- * Parse the byte[] and fill fieldStart, fieldLength,
- * fieldInited and fieldIsNull.
+ * Parse the byte[] and fill fieldStart, fieldLength, fieldInited and
+ * fieldIsNull.
*/
private void parse() {
-
- List extends StructField> fieldRefs = ((StructObjectInspector)oi).getAllStructFieldRefs();
-
- if (fields == null) {
+
+ List extends StructField> fieldRefs = ((StructObjectInspector) oi)
+ .getAllStructFieldRefs();
+
+ if (fields == null) {
fields = new LazyBinaryObject[fieldRefs.size()];
- for (int i = 0 ; i < fields.length; i++) {
- fields[i] = LazyBinaryFactory.createLazyBinaryObject(fieldRefs.get(i).getFieldObjectInspector());
+ for (int i = 0; i < fields.length; i++) {
+ fields[i] = LazyBinaryFactory.createLazyBinaryObject(fieldRefs.get(i)
+ .getFieldObjectInspector());
}
fieldInited = new boolean[fields.length];
fieldIsNull = new boolean[fields.length];
- fieldStart = new int[fields.length];
+ fieldStart = new int[fields.length];
fieldLength = new int[fields.length];
}
-
+
/**
- * Please note that one null byte is followed by eight fields,
- * then more null byte and fields.
+ * Please note that one null byte is followed by eight fields, then more
+ * null byte and fields.
*/
-
+
int fieldId = 0;
- int structByteEnd = start + length;
+ int structByteEnd = start + length;
byte[] bytes = this.bytes.getData();
- byte nullByte = bytes[start];
- int lastFieldByteEnd = start + 1;
+ byte nullByte = bytes[start];
+ int lastFieldByteEnd = start + 1;
// Go through all bytes in the byte[]
- for (int i=0; i structByteEnd) {
missingFieldWarned = true;
@@ -162,21 +165,22 @@
+ "only got " + fieldId + "! Ignoring similar problems.");
}
- Arrays.fill(fieldInited, false);
- parsed = true;
+ Arrays.fill(fieldInited, false);
+ parsed = true;
}
-
+
/**
* Get one field out of the struct.
*
- * If the field is a primitive field, return the actual object.
- * Otherwise return the LazyObject. This is because PrimitiveObjectInspector
- * does not have control over the object used by the user - the user simply
- * directly use the Object instead of going through
- * Object PrimitiveObjectInspector.get(Object).
+ * If the field is a primitive field, return the actual object. Otherwise
+ * return the LazyObject. This is because PrimitiveObjectInspector does not
+ * have control over the object used by the user - the user simply directly
+ * use the Object instead of going through Object
+ * PrimitiveObjectInspector.get(Object).
*
- * @param fieldID The field ID
- * @return The field as a LazyObject
+ * @param fieldID
+ * The field ID
+ * @return The field as a LazyObject
*/
public Object getField(int fieldID) {
if (!parsed) {
@@ -184,15 +188,17 @@
}
return uncheckedGetField(fieldID);
}
-
+
/**
- * Get the field out of the row without checking parsed.
- * This is called by both getField and getFieldsAsList.
- * @param fieldID The id of the field starting from 0.
- * @return The value of the field
+ * Get the field out of the row without checking parsed. This is called by
+ * both getField and getFieldsAsList.
+ *
+ * @param fieldID
+ * The id of the field starting from 0.
+ * @return The value of the field
*/
private Object uncheckedGetField(int fieldID) {
- // Test the length first so in most cases we avoid doing a byte[]
+ // Test the length first so in most cases we avoid doing a byte[]
// comparison.
if (fieldIsNull[fieldID]) {
return null;
@@ -205,8 +211,10 @@
}
ArrayList cachedList;
+
/**
* Get the values of the fields as an ArrayList.
+ *
* @return The values of the fields as an ArrayList.
*/
public ArrayList