Index: serde/src/java/org/apache/hadoop/hive/serde2/lazy/ByteArrayRef.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/lazy/ByteArrayRef.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/lazy/ByteArrayRef.java (working copy) @@ -20,14 +20,12 @@ /** * ByteArrayRef stores a reference to a byte array. * - * The LazyObject hierarchy uses a reference to a single ByteArrayRef, - * so that it's much faster to switch to the next row and release the - * reference to the old row (so that the system can do garbage collection - * if needed). + * The LazyObject hierarchy uses a reference to a single ByteArrayRef, so that + * it's much faster to switch to the next row and release the reference to the + * old row (so that the system can do garbage collection if needed). */ public class ByteArrayRef { - /** * Stores the actual data. */ @@ -40,5 +38,5 @@ public void setData(byte[] data) { this.data = data; } - + } Index: serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyLong.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyLong.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyLong.java (working copy) @@ -36,9 +36,9 @@ *

* */ -public class LazyLong extends LazyPrimitive { +public class LazyLong extends + LazyPrimitive { - public LazyLong(LazyLongObjectInspector oi) { super(oi); data = new LongWritable(); @@ -48,7 +48,7 @@ super(copy); data = new LongWritable(copy.data.get()); } - + @Override public void init(ByteArrayRef bytes, int start, int length) { try { @@ -67,38 +67,38 @@ * @param bytes * @param start * @param length - * a UTF-8 encoded string representation of a long quantity. + * a UTF-8 encoded string representation of a long quantity. * @return long the value represented by the argument * @exception NumberFormatException - * if the argument could not be parsed as a long quantity. + * if the argument could not be parsed as a long quantity. */ - public static long parseLong(byte[] bytes, int start, int length) throws NumberFormatException { + public static long parseLong(byte[] bytes, int start, int length) + throws NumberFormatException { return parseLong(bytes, start, length, 10); } /** * Parses the string argument as if it was an long value and returns the * result. Throws NumberFormatException if the string does not represent an - * long quantity. The second argument specifies the radix to use when - * parsing the value. + * long quantity. The second argument specifies the radix to use when parsing + * the value. * * @param bytes * @param start * @param length - * a UTF-8 encoded string representation of a long quantity. + * a UTF-8 encoded string representation of a long quantity. * @param radix - * the base to use for conversion. + * the base to use for conversion. * @return the value represented by the argument * @exception NumberFormatException - * if the argument could not be parsed as an long quantity. + * if the argument could not be parsed as an long quantity. */ public static long parseLong(byte[] bytes, int start, int length, int radix) throws NumberFormatException { if (bytes == null) { throw new NumberFormatException("String is null"); } - if (radix < Character.MIN_RADIX || - radix > Character.MAX_RADIX) { + if (radix < Character.MIN_RADIX || radix > Character.MAX_RADIX) { throw new NumberFormatException("Invalid radix: " + radix); } if (length == 0) { @@ -107,9 +107,10 @@ int offset = start; boolean negative = bytes[start] == '-'; if (negative || bytes[start] == '+') { - offset ++; + offset++; if (length == 1) { - throw new NumberFormatException(LazyUtils.convertToString(bytes, start, length)); + throw new NumberFormatException(LazyUtils.convertToString(bytes, start, + length)); } } @@ -117,61 +118,65 @@ } /** - /** - * Parses the string argument as if it was an long value and returns the + * /** Parses the string argument as if it was an long value and returns the * result. Throws NumberFormatException if the string does not represent an - * long quantity. The second argument specifies the radix to use when - * parsing the value. + * long quantity. The second argument specifies the radix to use when parsing + * the value. * * @param bytes * @param start * @param length - * a UTF-8 encoded string representation of a long quantity. + * a UTF-8 encoded string representation of a long quantity. * @param offset - * the starting position after the sign (if exists) + * the starting position after the sign (if exists) * @param radix - * the base to use for conversion. + * the base to use for conversion. * @param negative - * whether the number is negative. + * whether the number is negative. * @return the value represented by the argument * @exception NumberFormatException - * if the argument could not be parsed as an long quantity. + * if the argument could not be parsed as an long quantity. */ - private static long parse(byte[] bytes, int start, int length, int offset, int radix, - boolean negative) { + private static long parse(byte[] bytes, int start, int length, int offset, + int radix, boolean negative) { long max = Long.MIN_VALUE / radix; long result = 0, end = start + length; while (offset < end) { int digit = LazyUtils.digit(bytes[offset++], radix); if (digit == -1 || max > result) { - throw new NumberFormatException(LazyUtils.convertToString(bytes, start, length)); + throw new NumberFormatException(LazyUtils.convertToString(bytes, start, + length)); } long next = result * radix - digit; if (next > result) { - throw new NumberFormatException(LazyUtils.convertToString(bytes, start, length)); + throw new NumberFormatException(LazyUtils.convertToString(bytes, start, + length)); } result = next; } if (!negative) { result = -result; if (result < 0) { - throw new NumberFormatException(LazyUtils.convertToString(bytes, start, length)); + throw new NumberFormatException(LazyUtils.convertToString(bytes, start, + length)); } } return result; } /** - * Writes out the text representation of an integer using base 10 to an + * Writes out the text representation of an integer using base 10 to an * OutputStream in UTF-8 encoding. - * - * Note: division by a constant (like 10) is much faster than division by - * a variable. That's one of the reasons that we don't make radix a - * parameter here. - * - * @param out the outputstream to write to - * @param i an int to write out - * @throws IOException + * + * Note: division by a constant (like 10) is much faster than division by a + * variable. That's one of the reasons that we don't make radix a parameter + * here. + * + * @param out + * the outputstream to write to + * @param i + * an int to write out + * @throws IOException */ public static void writeUTF8(OutputStream out, long i) throws IOException { if (i == 0) { @@ -187,14 +192,14 @@ // of overflow here. i = -i; } - + long start = 1000000000000000000L; - while (i/start == 0) { + while (i / start == 0) { start /= 10; } - + while (start > 0) { - out.write('0' - (int)((i / start) % 10)); + out.write('0' - (int) ((i / start) % 10)); start /= 10; } } Index: serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyShort.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyShort.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyShort.java (working copy) @@ -33,18 +33,19 @@ *

* */ -public class LazyShort extends LazyPrimitive { +public class LazyShort extends + LazyPrimitive { public LazyShort(LazyShortObjectInspector oi) { super(oi); data = new ShortWritable(); } - + public LazyShort(LazyShort copy) { super(copy); data = new ShortWritable(copy.data.get()); } - + @Override public void init(ByteArrayRef bytes, int start, int length) { try { @@ -63,12 +64,12 @@ * @param bytes * @param start * @param length - * a UTF-8 encoded string representation of a short quantity. + * a UTF-8 encoded string representation of a short quantity. * @return short the value represented by the argument * @exception NumberFormatException - * if the argument could not be parsed as a short quantity. + * if the argument could not be parsed as a short quantity. */ - public static short parseShort(byte[] bytes, int start, int length) + public static short parseShort(byte[] bytes, int start, int length) throws NumberFormatException { return parseShort(bytes, start, length, 10); } @@ -76,18 +77,18 @@ /** * Parses the string argument as if it was a short value and returns the * result. Throws NumberFormatException if the string does not represent a - * single short quantity. The second argument specifies the radix to use - * when parsing the value. + * single short quantity. The second argument specifies the radix to use when + * parsing the value. * * @param bytes * @param start * @param length - * a UTF-8 encoded string representation of a short quantity. + * a UTF-8 encoded string representation of a short quantity. * @param radix - * the radix to use when parsing. + * the radix to use when parsing. * @return short the value represented by the argument * @exception NumberFormatException - * if the argument could not be parsed as a short quantity. + * if the argument could not be parsed as a short quantity. */ public static short parseShort(byte[] bytes, int start, int length, int radix) throws NumberFormatException { Index: serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyUtils.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyUtils.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyUtils.java (working copy) @@ -37,32 +37,38 @@ /** * Returns the digit represented by character b. - * @param b The ascii code of the character - * @param radix The radix - * @return -1 if it's invalid + * + * @param b + * The ascii code of the character + * @param radix + * The radix + * @return -1 if it's invalid */ public static int digit(int b, int radix) { int r = -1; - if (b >= '0' && b<='9') { + if (b >= '0' && b <= '9') { r = b - '0'; - } else if (b >= 'A' && b<='Z') { + } else if (b >= 'A' && b <= 'Z') { r = b - 'A' + 10; } else if (b >= 'a' && b <= 'z') { r = b - 'a' + 10; } - if (r >= radix) r = -1; + if (r >= radix) { + r = -1; + } return r; } - + /** - * Returns -1 if the first byte sequence is lexicographically less than the second; - * returns +1 if the second byte sequence is lexicographically less than the first; - * otherwise return 0. + * Returns -1 if the first byte sequence is lexicographically less than the + * second; returns +1 if the second byte sequence is lexicographically less + * than the first; otherwise return 0. */ - public static int compare(byte[] b1, int start1, int length1, byte[] b2, int start2, int length2) { - + public static int compare(byte[] b1, int start1, int length1, byte[] b2, + int start2, int length2) { + int min = Math.min(length1, length2); - + for (int i = 0; i < min; i++) { if (b1[start1 + i] == b2[start2 + i]) { continue; @@ -71,19 +77,27 @@ return -1; } else { return 1; - } + } } - - if (length1 < length2) return -1; - if (length1 > length2) return 1; + + if (length1 < length2) { + return -1; + } + if (length1 > length2) { + return 1; + } return 0; } - + /** * Convert a UTF-8 byte array to String. - * @param bytes The byte[] containing the UTF-8 String. - * @param start The start position inside the bytes. - * @param length The length of the data, starting from "start" + * + * @param bytes + * The byte[] containing the UTF-8 String. + * @param start + * The start position inside the bytes. + * @param length + * The length of the data, starting from "start" * @return The unicode String */ public static String convertToString(byte[] bytes, int start, int length) { @@ -94,19 +108,22 @@ } } + static byte[] trueBytes = { (byte) 't', 'r', 'u', 'e' }; + static byte[] falseBytes = { (byte) 'f', 'a', 'l', 's', 'e' }; - static byte[] trueBytes = {(byte)'t', 'r', 'u', 'e'}; - static byte[] falseBytes = {(byte)'f', 'a', 'l', 's', 'e'}; - /** * Write the bytes with special characters escaped. - * @param escaped Whether the data should be written out in an escaped way. - * @param escapeChar if escaped, the char for prefixing special characters. - * @param needsEscape if escaped, whether a specific character needs escaping. - * This array should have size of 128. + * + * @param escaped + * Whether the data should be written out in an escaped way. + * @param escapeChar + * if escaped, the char for prefixing special characters. + * @param needsEscape + * if escaped, whether a specific character needs escaping. This + * array should have size of 128. */ - private static void writeEscaped(OutputStream out, byte[] bytes, - int start, int len, boolean escaped, byte escapeChar, boolean[] needsEscape) + private static void writeEscaped(OutputStream out, byte[] bytes, int start, + int len, boolean escaped, byte escapeChar, boolean[] needsEscape) throws IOException { if (escaped) { int end = start + len; @@ -126,71 +143,79 @@ out.write(bytes, 0, len); } } - - + /** - * Write out the text representation of a Primitive Object to a UTF8 byte stream. - * @param out The UTF8 byte OutputStream - * @param o The primitive Object - * @param needsEscape Whether a character needs escaping. This array should have size of 128. + * Write out the text representation of a Primitive Object to a UTF8 byte + * stream. + * + * @param out + * The UTF8 byte OutputStream + * @param o + * The primitive Object + * @param needsEscape + * Whether a character needs escaping. This array should have size of + * 128. */ - public static void writePrimitiveUTF8(OutputStream out, Object o, PrimitiveObjectInspector oi, - boolean escaped, byte escapeChar, boolean[] needsEscape) throws IOException { - + public static void writePrimitiveUTF8(OutputStream out, Object o, + PrimitiveObjectInspector oi, boolean escaped, byte escapeChar, + boolean[] needsEscape) throws IOException { + switch (oi.getPrimitiveCategory()) { - case BOOLEAN: { - boolean b = ((BooleanObjectInspector)oi).get(o); - if (b) { - out.write(trueBytes, 0, trueBytes.length); - } else { - out.write(falseBytes, 0, falseBytes.length); - } - break; + case BOOLEAN: { + boolean b = ((BooleanObjectInspector) oi).get(o); + if (b) { + out.write(trueBytes, 0, trueBytes.length); + } else { + out.write(falseBytes, 0, falseBytes.length); } - case BYTE: { - LazyInteger.writeUTF8(out, ((ByteObjectInspector)oi).get(o)); - break; - } - case SHORT: { - LazyInteger.writeUTF8(out, ((ShortObjectInspector)oi).get(o)); - break; - } - case INT: { - LazyInteger.writeUTF8(out, ((IntObjectInspector)oi).get(o)); - break; - } - case LONG: { - LazyLong.writeUTF8(out, ((LongObjectInspector)oi).get(o)); - break; - } - case FLOAT: { - float f = ((FloatObjectInspector)oi).get(o); - ByteBuffer b = Text.encode(String.valueOf(f)); - out.write(b.array(), 0, b.limit()); - break; - } - case DOUBLE: { - double d = ((DoubleObjectInspector)oi).get(o); - ByteBuffer b = Text.encode(String.valueOf(d)); - out.write(b.array(), 0, b.limit()); - break; - } - case STRING: { - Text t = ((StringObjectInspector)oi).getPrimitiveWritableObject(o); - writeEscaped(out, t.getBytes(), 0, t.getLength(), escaped, escapeChar, needsEscape); - break; - } - default: { - throw new RuntimeException("Hive internal error."); - } + break; } + case BYTE: { + LazyInteger.writeUTF8(out, ((ByteObjectInspector) oi).get(o)); + break; + } + case SHORT: { + LazyInteger.writeUTF8(out, ((ShortObjectInspector) oi).get(o)); + break; + } + case INT: { + LazyInteger.writeUTF8(out, ((IntObjectInspector) oi).get(o)); + break; + } + case LONG: { + LazyLong.writeUTF8(out, ((LongObjectInspector) oi).get(o)); + break; + } + case FLOAT: { + float f = ((FloatObjectInspector) oi).get(o); + ByteBuffer b = Text.encode(String.valueOf(f)); + out.write(b.array(), 0, b.limit()); + break; + } + case DOUBLE: { + double d = ((DoubleObjectInspector) oi).get(o); + ByteBuffer b = Text.encode(String.valueOf(d)); + out.write(b.array(), 0, b.limit()); + break; + } + case STRING: { + Text t = ((StringObjectInspector) oi).getPrimitiveWritableObject(o); + writeEscaped(out, t.getBytes(), 0, t.getLength(), escaped, escapeChar, + needsEscape); + break; + } + default: { + throw new RuntimeException("Hive internal error."); + } + } } - + public static int hashBytes(byte[] data, int start, int len) { int hash = 1; - for (int i = start; i < len; i++) - hash = (31 * hash) + (int)data[i]; + for (int i = start; i < len; i++) { + hash = (31 * hash) + data[i]; + } return hash; } - + } Index: serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyByte.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyByte.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyByte.java (working copy) @@ -33,7 +33,8 @@ *

* */ -public class LazyByte extends LazyPrimitive { +public class LazyByte extends + LazyPrimitive { public LazyByte(LazyByteObjectInspector oi) { super(oi); @@ -44,7 +45,7 @@ super(copy); data = new ByteWritable(copy.data.get()); } - + @Override public void init(ByteArrayRef bytes, int start, int length) { try { @@ -54,7 +55,7 @@ isNull = true; } } - + /** * Parses the string argument as if it was a byte value and returns the * result. Throws NumberFormatException if the string does not represent a @@ -63,15 +64,16 @@ * @param bytes * @param start * @param length - * a UTF-8 encoded string representation of a single byte quantity. + * a UTF-8 encoded string representation of a single byte quantity. * @return byte the value represented by the argument * @throws NumberFormatException - * if the argument could not be parsed as a byte quantity. + * if the argument could not be parsed as a byte quantity. */ - public static byte parseByte(byte[] bytes, int start, int length) throws NumberFormatException { - return parseByte(bytes, start, length, 10); + public static byte parseByte(byte[] bytes, int start, int length) + throws NumberFormatException { + return parseByte(bytes, start, length, 10); } - + /** * Parses the string argument as if it was a byte value and returns the * result. Throws NumberFormatException if the string does not represent a @@ -81,22 +83,21 @@ * @param bytes * @param start * @param length - * a UTF-8 encoded string representation of a single byte quantity. + * a UTF-8 encoded string representation of a single byte quantity. * @param radix - * the radix to use when parsing. + * the radix to use when parsing. * @return byte the value represented by the argument * @throws NumberFormatException - * if the argument could not be parsed as a byte quantity. + * if the argument could not be parsed as a byte quantity. */ public static byte parseByte(byte[] bytes, int start, int length, int radix) - throws NumberFormatException { - int intValue = LazyInteger.parseInt(bytes, start, length, radix); - byte result = (byte) intValue; - if (result == intValue) { - return result; - } - throw new NumberFormatException(); + throws NumberFormatException { + int intValue = LazyInteger.parseInt(bytes, start, length, radix); + byte result = (byte) intValue; + if (result == intValue) { + return result; + } + throw new NumberFormatException(); } - } Index: serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyFloat.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyFloat.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyFloat.java (working copy) @@ -27,18 +27,19 @@ * LazyObject for storing a value of Double. * */ -public class LazyFloat extends LazyPrimitive { +public class LazyFloat extends + LazyPrimitive { public LazyFloat(LazyFloatObjectInspector oi) { super(oi); data = new FloatWritable(); } - + public LazyFloat(LazyFloat copy) { super(copy); data = new FloatWritable(copy.data.get()); - } - + } + @Override public void init(ByteArrayRef bytes, int start, int length) { try { Index: serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyObject.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyObject.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyObject.java (working copy) @@ -22,40 +22,45 @@ /** * LazyObject stores an object in a range of bytes in a byte[]. * - * A LazyObject can represent any primitive object or hierarchical object - * like array, map or struct. + * A LazyObject can represent any primitive object or hierarchical object like + * array, map or struct. */ public abstract class LazyObject { OI oi; - + /** * Create a LazyObject. - * @param oi Derived classes can access meta information about this Lazy - * Object (e.g, separator, nullSequence, escaper) from it. + * + * @param oi + * Derived classes can access meta information about this Lazy Object + * (e.g, separator, nullSequence, escaper) from it. */ protected LazyObject(OI oi) { this.oi = oi; } - + /** - * Set the data for this LazyObject. - * We take ByteArrayRef instead of byte[] so that we will be able to drop - * the reference to byte[] by a single assignment. - * The ByteArrayRef object can be reused across multiple rows. - * @param bytes The wrapper of the byte[]. - * @param start The start position inside the bytes. - * @param length The length of the data, starting from "start" + * Set the data for this LazyObject. We take ByteArrayRef instead of byte[] so + * that we will be able to drop the reference to byte[] by a single + * assignment. The ByteArrayRef object can be reused across multiple rows. + * + * @param bytes + * The wrapper of the byte[]. + * @param start + * The start position inside the bytes. + * @param length + * The length of the data, starting from "start" * @see ByteArrayRef */ public abstract void init(ByteArrayRef bytes, int start, int length); /** - * If the LazyObject is a primitive Object, then deserialize it and return - * the actual primitive Object. - * Otherwise (array, map, struct), return this. + * If the LazyObject is a primitive Object, then deserialize it and return the + * actual primitive Object. Otherwise (array, map, struct), return this. */ public abstract Object getObject(); - + + @Override public abstract int hashCode(); } Index: serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyString.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyString.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyString.java (working copy) @@ -34,24 +34,24 @@ super(copy); data = new Text(copy.data); } - + @Override public void init(ByteArrayRef bytes, int start, int length) { if (oi.isEscaped()) { byte escapeChar = oi.getEscapeChar(); byte[] inputBytes = bytes.getData(); - + // First calculate the length of the output string int outputLength = 0; - for (int i=0; i createLazyPrimitiveClass(PrimitiveObjectInspector oi) { + public static LazyPrimitive createLazyPrimitiveClass( + PrimitiveObjectInspector oi) { PrimitiveCategory p = oi.getPrimitiveCategory(); - switch(p) { - case BOOLEAN: { - return new LazyBoolean((LazyBooleanObjectInspector)oi); - } - case BYTE: { - return new LazyByte((LazyByteObjectInspector)oi); - } - case SHORT: { - return new LazyShort((LazyShortObjectInspector)oi); - } - case INT: { - return new LazyInteger((LazyIntObjectInspector)oi); - } - case LONG: { - return new LazyLong((LazyLongObjectInspector)oi); - } - case FLOAT: { - return new LazyFloat((LazyFloatObjectInspector)oi); - } - case DOUBLE: { - return new LazyDouble((LazyDoubleObjectInspector)oi); - } - case STRING: { - return new LazyString((LazyStringObjectInspector)oi); - } - default: { - throw new RuntimeException("Internal error: no LazyObject for " + p); - } + switch (p) { + case BOOLEAN: { + return new LazyBoolean((LazyBooleanObjectInspector) oi); } + case BYTE: { + return new LazyByte((LazyByteObjectInspector) oi); + } + case SHORT: { + return new LazyShort((LazyShortObjectInspector) oi); + } + case INT: { + return new LazyInteger((LazyIntObjectInspector) oi); + } + case LONG: { + return new LazyLong((LazyLongObjectInspector) oi); + } + case FLOAT: { + return new LazyFloat((LazyFloatObjectInspector) oi); + } + case DOUBLE: { + return new LazyDouble((LazyDoubleObjectInspector) oi); + } + case STRING: { + return new LazyString((LazyStringObjectInspector) oi); + } + default: { + throw new RuntimeException("Internal error: no LazyObject for " + p); + } + } } /** @@ -96,70 +88,75 @@ */ public static LazyObject createLazyObject(ObjectInspector oi) { ObjectInspector.Category c = oi.getCategory(); - switch(c) { + switch (c) { case PRIMITIVE: - return createLazyPrimitiveClass((PrimitiveObjectInspector)oi); + return createLazyPrimitiveClass((PrimitiveObjectInspector) oi); case MAP: - return new LazyMap((LazyMapObjectInspector)oi); - case LIST: - return new LazyArray((LazyListObjectInspector)oi); + return new LazyMap((LazyMapObjectInspector) oi); + case LIST: + return new LazyArray((LazyListObjectInspector) oi); case STRUCT: - return new LazyStruct((LazySimpleStructObjectInspector)oi); + return new LazyStruct((LazySimpleStructObjectInspector) oi); } throw new RuntimeException("Hive LazySerDe Internal error."); } - + /** * Create a hierarchical ObjectInspector for LazyObject with the given * typeInfo. - * @param typeInfo The type information for the LazyObject - * @param separator The array of separators for delimiting each level - * @param separatorIndex The current level (for separators). List(array), - * struct uses 1 level of separator, and map uses 2 - * levels: the first one for delimiting entries, the - * second one for delimiting key and values. - * @param nullSequence The sequence of bytes representing NULL. - * @return The ObjectInspector + * + * @param typeInfo + * The type information for the LazyObject + * @param separator + * The array of separators for delimiting each level + * @param separatorIndex + * The current level (for separators). List(array), struct uses 1 + * level of separator, and map uses 2 levels: the first one for + * delimiting entries, the second one for delimiting key and values. + * @param nullSequence + * The sequence of bytes representing NULL. + * @return The ObjectInspector */ - public static ObjectInspector createLazyObjectInspector(TypeInfo typeInfo, byte[] separator, - int separatorIndex, Text nullSequence, boolean escaped, byte escapeChar) { + public static ObjectInspector createLazyObjectInspector(TypeInfo typeInfo, + byte[] separator, int separatorIndex, Text nullSequence, boolean escaped, + byte escapeChar) { ObjectInspector.Category c = typeInfo.getCategory(); - switch(c) { + switch (c) { case PRIMITIVE: return LazyPrimitiveObjectInspectorFactory.getLazyObjectInspector( - ((PrimitiveTypeInfo)typeInfo).getPrimitiveCategory(), escaped, escapeChar); + ((PrimitiveTypeInfo) typeInfo).getPrimitiveCategory(), escaped, + escapeChar); case MAP: return LazyObjectInspectorFactory.getLazySimpleMapObjectInspector( - createLazyObjectInspector(((MapTypeInfo)typeInfo).getMapKeyTypeInfo(), - separator, separatorIndex+2, nullSequence, escaped, escapeChar), - createLazyObjectInspector(((MapTypeInfo)typeInfo).getMapValueTypeInfo(), - separator, separatorIndex+2, nullSequence, escaped, escapeChar), - separator[separatorIndex], - separator[separatorIndex+1], + createLazyObjectInspector(((MapTypeInfo) typeInfo) + .getMapKeyTypeInfo(), separator, separatorIndex + 2, + nullSequence, escaped, escapeChar), createLazyObjectInspector( + ((MapTypeInfo) typeInfo).getMapValueTypeInfo(), separator, + separatorIndex + 2, nullSequence, escaped, escapeChar), + separator[separatorIndex], separator[separatorIndex + 1], nullSequence, escaped, escapeChar); - case LIST: + case LIST: return LazyObjectInspectorFactory.getLazySimpleListObjectInspector( - createLazyObjectInspector(((ListTypeInfo)typeInfo).getListElementTypeInfo(), - separator, separatorIndex+1, nullSequence, escaped, escapeChar), - separator[separatorIndex], + createLazyObjectInspector(((ListTypeInfo) typeInfo) + .getListElementTypeInfo(), separator, separatorIndex + 1, + nullSequence, escaped, escapeChar), separator[separatorIndex], nullSequence, escaped, escapeChar); case STRUCT: - StructTypeInfo structTypeInfo = (StructTypeInfo)typeInfo; + StructTypeInfo structTypeInfo = (StructTypeInfo) typeInfo; List fieldNames = structTypeInfo.getAllStructFieldNames(); - List fieldTypeInfos = structTypeInfo.getAllStructFieldTypeInfos(); - List fieldObjectInspectors = new ArrayList(fieldTypeInfos.size()); - for(int i=0; i fieldTypeInfos = structTypeInfo + .getAllStructFieldTypeInfos(); + List fieldObjectInspectors = new ArrayList( + fieldTypeInfos.size()); + for (int i = 0; i < fieldTypeInfos.size(); i++) { + fieldObjectInspectors.add(createLazyObjectInspector(fieldTypeInfos + .get(i), separator, separatorIndex + 1, nullSequence, escaped, + escapeChar)); } return LazyObjectInspectorFactory.getLazySimpleStructObjectInspector( - fieldNames, - fieldObjectInspectors, - separator[separatorIndex], - nullSequence, - false, escaped, escapeChar); + fieldNames, fieldObjectInspectors, separator[separatorIndex], + nullSequence, false, escaped, escapeChar); } throw new RuntimeException("Hive LazySerDe Internal error."); @@ -169,29 +166,33 @@ * Create a hierarchical ObjectInspector for LazyStruct with the given * columnNames and columnTypeInfos. * - * @param lastColumnTakesRest whether the last column of the struct should take - * the rest of the row if there are extra fields. - * @see LazyFactory#createLazyObjectInspector(TypeInfo, byte[], int, Text, boolean, byte) - */ - public static ObjectInspector createLazyStructInspector(List columnNames, - List typeInfos, byte[] separators, - Text nullSequence, boolean lastColumnTakesRest, boolean escaped, byte escapeChar) { - ArrayList columnObjectInspectors = - new ArrayList(typeInfos.size()); - for (int i=0; i columnNames, List typeInfos, byte[] separators, + Text nullSequence, boolean lastColumnTakesRest, boolean escaped, + byte escapeChar) { + ArrayList columnObjectInspectors = new ArrayList( + typeInfos.size()); + for (int i = 0; i < typeInfos.size(); i++) { + columnObjectInspectors.add(LazyFactory.createLazyObjectInspector( + typeInfos.get(i), separators, 1, nullSequence, escaped, escapeChar)); } - return - LazyObjectInspectorFactory.getLazySimpleStructObjectInspector(columnNames, - columnObjectInspectors, separators[0], nullSequence, lastColumnTakesRest, escaped, escapeChar); + return LazyObjectInspectorFactory.getLazySimpleStructObjectInspector( + columnNames, columnObjectInspectors, separators[0], nullSequence, + lastColumnTakesRest, escaped, escapeChar); } - + /** * Create a hierarchical ObjectInspector for ColumnarStruct with the given * columnNames and columnTypeInfos. * - * @see LazyFactory#createLazyObjectInspector(TypeInfo, byte[], int, Text, boolean, byte) + * @see LazyFactory#createLazyObjectInspector(TypeInfo, byte[], int, Text, + * boolean, byte) */ public static ObjectInspector createColumnarStructInspector( List columnNames, List columnTypes, byte[] separators, @@ -199,11 +200,12 @@ ArrayList columnObjectInspectors = new ArrayList( columnTypes.size()); for (int i = 0; i < columnTypes.size(); i++) { - columnObjectInspectors.add(LazyFactory.createLazyObjectInspector( - columnTypes.get(i), separators, 1, nullSequence, escaped, escapeChar)); + columnObjectInspectors + .add(LazyFactory.createLazyObjectInspector(columnTypes.get(i), + separators, 1, nullSequence, escaped, escapeChar)); } return ObjectInspectorFactory.getColumnarStructObjectInspector(columnNames, columnObjectInspectors, nullSequence); } - + } Index: serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyPrimitive.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyPrimitive.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyPrimitive.java (working copy) @@ -23,8 +23,8 @@ /** * LazyPrimitive stores a primitive Object in a LazyObject. */ -public abstract class LazyPrimitive extends LazyObject { +public abstract class LazyPrimitive + extends LazyObject { LazyPrimitive(OI oi) { super(oi); @@ -39,9 +39,10 @@ boolean isNull = false; /** - * Returns the primitive object represented by this LazyObject. - * This is useful because it can make sure we have "null" for null objects. + * Returns the primitive object represented by this LazyObject. This is useful + * because it can make sure we have "null" for null objects. */ + @Override public Object getObject() { return isNull ? null : this; } @@ -49,13 +50,15 @@ public T getWritableObject() { return isNull ? null : data; } - + + @Override public String toString() { return isNull ? null : data.toString(); } - - public int hashCode(){ + + @Override + public int hashCode() { return isNull ? 0 : data.hashCode(); } - + } Index: serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyDouble.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyDouble.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyDouble.java (working copy) @@ -27,7 +27,8 @@ * LazyObject for storing a value of Double. * */ -public class LazyDouble extends LazyPrimitive { +public class LazyDouble extends + LazyPrimitive { public LazyDouble(LazyDoubleObjectInspector oi) { super(oi); @@ -37,8 +38,8 @@ public LazyDouble(LazyDouble copy) { super(copy); data = new DoubleWritable(copy.data.get()); - } - + } + @Override public void init(ByteArrayRef bytes, int start, int length) { try { Index: serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySimpleSerDe.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySimpleSerDe.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySimpleSerDe.java (working copy) @@ -50,25 +50,26 @@ import org.apache.hadoop.io.Writable; /** - * LazySimpleSerDe can be used to read the same data format as + * LazySimpleSerDe can be used to read the same data format as * MetadataTypedColumnsetSerDe and TCTLSeparatedProtocol. * - * However, LazySimpleSerDe creates Objects in a lazy way, to - * provide better performance. + * However, LazySimpleSerDe creates Objects in a lazy way, to provide better + * performance. * - * Also LazySimpleSerDe outputs typed columns instead of treating - * all columns as String like MetadataTypedColumnsetSerDe. + * Also LazySimpleSerDe outputs typed columns instead of treating all columns as + * String like MetadataTypedColumnsetSerDe. */ public class LazySimpleSerDe implements SerDe { - public static final Log LOG = LogFactory.getLog( - LazySimpleSerDe.class.getName()); + public static final Log LOG = LogFactory.getLog(LazySimpleSerDe.class + .getName()); - final public static byte[] DefaultSeparators = {(byte)1, (byte)2, (byte)3}; + final public static byte[] DefaultSeparators = { (byte) 1, (byte) 2, (byte) 3 }; private ObjectInspector cachedObjectInspector; - private boolean useJSONSerialize; // use json to serialize + private boolean useJSONSerialize; // use json to serialize + @Override public String toString() { return getClass().toString() + "[" @@ -85,16 +86,19 @@ /** * Return the byte value of the number string. - * @param altValue The string containing a number. - * @param defaultVal If the altValue does not represent a number, - * return the defaultVal. + * + * @param altValue + * The string containing a number. + * @param defaultVal + * If the altValue does not represent a number, return the + * defaultVal. */ public static byte getByte(String altValue, byte defaultVal) { if (altValue != null && altValue.length() > 0) { try { return Byte.valueOf(altValue).byteValue(); - } catch(NumberFormatException e) { - return (byte)altValue.charAt(0); + } catch (NumberFormatException e) { + return (byte) altValue.charAt(0); } } return defaultVal; @@ -108,12 +112,12 @@ boolean lastColumnTakesRest; List columnNames; List columnTypes; - + boolean escaped; byte escapeChar; boolean[] needsEscape; boolean jsonSerialize; - + public List getColumnTypes() { return columnTypes; } @@ -121,7 +125,7 @@ public List getColumnNames() { return columnNames; } - + public byte[] getSeparators() { return separators; } @@ -141,13 +145,15 @@ public boolean isLastColumnTakesRest() { return lastColumnTakesRest; } - + public boolean isEscaped() { return escaped; } + public byte getEscapeChar() { return escapeChar; } + public boolean[] getNeedsEscape() { return needsEscape; } @@ -160,44 +166,45 @@ } /** - * @param jsonSerialize the jsonSerialize to set + * @param jsonSerialize + * the jsonSerialize to set */ public void setJsonSerialize(boolean jsonSerialize) { this.jsonSerialize = jsonSerialize; } - + } SerDeParameters serdeParams = null; /** - * Initialize the SerDe given the parameters. - * serialization.format: separator char or byte code (only supports - * byte-value up to 127) - * columns: ","-separated column names - * columns.types: ",", ":", or ";"-separated column types - * @see SerDe#initialize(Configuration, Properties) + * Initialize the SerDe given the parameters. serialization.format: separator + * char or byte code (only supports byte-value up to 127) columns: + * ","-separated column names columns.types: ",", ":", or ";"-separated column + * types + * + * @see SerDe#initialize(Configuration, Properties) */ - public void initialize(Configuration job, Properties tbl) - throws SerDeException { + public void initialize(Configuration job, Properties tbl) + throws SerDeException { serdeParams = LazySimpleSerDe.initSerdeParams(job, tbl, getClass() .getName()); // Create the ObjectInspectors for the fields - cachedObjectInspector = LazyFactory.createLazyStructInspector( - serdeParams.getColumnNames(), serdeParams.getColumnTypes(), - serdeParams.getSeparators(), serdeParams.getNullSequence(), - serdeParams.isLastColumnTakesRest(), - serdeParams.isEscaped(), - serdeParams.getEscapeChar()); + cachedObjectInspector = LazyFactory.createLazyStructInspector(serdeParams + .getColumnNames(), serdeParams.getColumnTypes(), serdeParams + .getSeparators(), serdeParams.getNullSequence(), serdeParams + .isLastColumnTakesRest(), serdeParams.isEscaped(), serdeParams + .getEscapeChar()); - if (serdeParams.isJsonSerialize()) + if (serdeParams.isJsonSerialize()) { setUseJSONSerialize(true); - + } + cachedLazyStruct = (LazyStruct) LazyFactory - .createLazyObject(cachedObjectInspector); - + .createLazyObject(cachedObjectInspector); + LOG.debug("LazySimpleSerDe initialized with: columnNames=" + serdeParams.columnNames + " columnTypes=" + serdeParams.columnTypes + " separator=" + Arrays.asList(serdeParams.separators) @@ -232,8 +239,9 @@ .equalsIgnoreCase("true")); String useJsonSerialize = tbl - .getProperty(Constants.SERIALIZATION_USE_JSON_OBJECTS); - serdeParams.jsonSerialize = (useJsonSerialize != null && useJsonSerialize.equalsIgnoreCase("true")); + .getProperty(Constants.SERIALIZATION_USE_JSON_OBJECTS); + serdeParams.jsonSerialize = (useJsonSerialize != null && useJsonSerialize + .equalsIgnoreCase("true")); // Read the configuration parameters String columnNameProperty = tbl.getProperty(Constants.LIST_COLUMNS); @@ -251,8 +259,9 @@ // Default type: all string StringBuilder sb = new StringBuilder(); for (int i = 0; i < serdeParams.columnNames.size(); i++) { - if (i > 0) + if (i > 0) { sb.append(":"); + } sb.append(Constants.STRING_TYPE_NAME); } columnTypeProperty = sb.toString(); @@ -271,37 +280,39 @@ // Create the LazyObject for storing the rows serdeParams.rowTypeInfo = TypeInfoFactory.getStructTypeInfo( serdeParams.columnNames, serdeParams.columnTypes); - + // Get the escape information String escapeProperty = tbl.getProperty(Constants.ESCAPE_CHAR); serdeParams.escaped = (escapeProperty != null); if (serdeParams.escaped) { - serdeParams.escapeChar = getByte(escapeProperty, (byte)'\\'); + serdeParams.escapeChar = getByte(escapeProperty, (byte) '\\'); } if (serdeParams.escaped) { serdeParams.needsEscape = new boolean[128]; - for (int i=0; i<128; i++) { + for (int i = 0; i < 128; i++) { serdeParams.needsEscape[i] = false; } serdeParams.needsEscape[serdeParams.escapeChar] = true; - for (int i=0; i getSerializedClass() { return Text.class; } - + Text serializeCache = new Text(); ByteStream.Output serializeStream = new ByteStream.Output(); + /** * Serialize a row of data. - * @param obj The row object - * @param objInspector The ObjectInspector for the row object - * @return The serialized Writable object - * @throws IOException - * @see SerDe#serialize(Object, ObjectInspector) + * + * @param obj + * The row object + * @param objInspector + * The ObjectInspector for the row object + * @return The serialized Writable object + * @throws IOException + * @see SerDe#serialize(Object, ObjectInspector) */ - public Writable serialize(Object obj, ObjectInspector objInspector) + public Writable serialize(Object obj, ObjectInspector objInspector) throws SerDeException { if (objInspector.getCategory() != Category.STRUCT) { - throw new SerDeException(getClass().toString() - + " can only serialize struct types, but we got: " + throw new SerDeException(getClass().toString() + + " can only serialize struct types, but we got: " + objInspector.getTypeName()); } // Prepare the field ObjectInspectors - StructObjectInspector soi = (StructObjectInspector)objInspector; + StructObjectInspector soi = (StructObjectInspector) objInspector; List fields = soi.getAllStructFieldRefs(); List list = soi.getStructFieldsDataAsList(obj); - List declaredFields =(serdeParams.rowTypeInfo != null && ((StructTypeInfo) serdeParams.rowTypeInfo) - .getAllStructFieldNames().size()>0)? ((StructObjectInspector)getObjectInspector()) + List declaredFields = (serdeParams.rowTypeInfo != null && ((StructTypeInfo) serdeParams.rowTypeInfo) + .getAllStructFieldNames().size() > 0) ? ((StructObjectInspector) getObjectInspector()) .getAllStructFieldRefs() : null; @@ -372,9 +387,9 @@ try { // Serialize each field - for (int i=0; i0) { + if (i > 0) { serializeStream.write(serdeParams.separators[0]); } // Get the field objectInspector and the field object. @@ -382,23 +397,22 @@ Object f = (list == null ? null : list.get(i)); if (declaredFields != null && i >= declaredFields.size()) { - throw new SerDeException( - "Error: expecting " + declaredFields.size() + throw new SerDeException("Error: expecting " + declaredFields.size() + " but asking for field " + i + "\n" + "data=" + obj + "\n" + "tableType=" + serdeParams.rowTypeInfo.toString() + "\n" - + "dataType=" + + "dataType=" + TypeInfoUtils.getTypeInfoFromObjectInspector(objInspector)); } - - // If the field that is passed in is NOT a primitive, and either the - // field is not declared (no schema was given at initialization), or - // the field is declared as a primitive in initialization, serialize - // the data to JSON string. Otherwise serialize the data in the + + // If the field that is passed in is NOT a primitive, and either the + // field is not declared (no schema was given at initialization), or + // the field is declared as a primitive in initialization, serialize + // the data to JSON string. Otherwise serialize the data in the // delimited way. if (!foi.getCategory().equals(Category.PRIMITIVE) - && (declaredFields == null || - declaredFields.get(i).getFieldObjectInspector().getCategory() - .equals(Category.PRIMITIVE) || useJSONSerialize)) { + && (declaredFields == null + || declaredFields.get(i).getFieldObjectInspector() + .getCategory().equals(Category.PRIMITIVE) || useJSONSerialize)) { serialize(serializeStream, SerDeUtils.getJSONString(f, foi), PrimitiveObjectInspectorFactory.javaStringObjectInspector, serdeParams.separators, 1, serdeParams.nullSequence, @@ -406,8 +420,8 @@ serdeParams.needsEscape); } else { serialize(serializeStream, f, foi, serdeParams.separators, 1, - serdeParams.nullSequence, serdeParams.escaped, serdeParams.escapeChar, - serdeParams.needsEscape); + serdeParams.nullSequence, serdeParams.escaped, + serdeParams.escapeChar, serdeParams.needsEscape); } } } catch (IOException e) { @@ -415,105 +429,119 @@ } // TODO: The copy of data is unnecessary, but there is no work-around // since we cannot directly set the private byte[] field inside Text. - serializeCache.set(serializeStream.getData(), 0, - serializeStream.getCount()); + serializeCache + .set(serializeStream.getData(), 0, serializeStream.getCount()); return serializeCache; } /** * Serialize the row into the StringBuilder. - * @param out The StringBuilder to store the serialized data. - * @param obj The object for the current field. - * @param objInspector The ObjectInspector for the current Object. - * @param separators The separators array. - * @param level The current level of separator. - * @param nullSequence The byte sequence representing the NULL value. - * @param escaped Whether we need to escape the data when writing out - * @param escapeChar Which char to use as the escape char, e.g. '\\' - * @param needsEscape Which chars needs to be escaped. This array should have size of 128. - * Negative byte values (or byte values >= 128) are never escaped. - * @throws IOException + * + * @param out + * The StringBuilder to store the serialized data. + * @param obj + * The object for the current field. + * @param objInspector + * The ObjectInspector for the current Object. + * @param separators + * The separators array. + * @param level + * The current level of separator. + * @param nullSequence + * The byte sequence representing the NULL value. + * @param escaped + * Whether we need to escape the data when writing out + * @param escapeChar + * Which char to use as the escape char, e.g. '\\' + * @param needsEscape + * Which chars needs to be escaped. This array should have size of + * 128. Negative byte values (or byte values >= 128) are never + * escaped. + * @throws IOException */ - public static void serialize(ByteStream.Output out, Object obj, + public static void serialize(ByteStream.Output out, Object obj, ObjectInspector objInspector, byte[] separators, int level, - Text nullSequence, boolean escaped, byte escapeChar, boolean[] needsEscape) throws IOException { - + Text nullSequence, boolean escaped, byte escapeChar, boolean[] needsEscape) + throws IOException { + if (obj == null) { out.write(nullSequence.getBytes(), 0, nullSequence.getLength()); return; } - + switch (objInspector.getCategory()) { - case PRIMITIVE: { - LazyUtils.writePrimitiveUTF8(out, obj, (PrimitiveObjectInspector)objInspector, escaped, escapeChar, needsEscape); - return; - } - case LIST: { - char separator = (char)separators[level]; - ListObjectInspector loi = (ListObjectInspector)objInspector; - List list = loi.getList(obj); - ObjectInspector eoi = loi.getListElementObjectInspector(); - if (list == null) { - out.write(nullSequence.getBytes(), 0, nullSequence.getLength()); - } else { - for (int i=0; i0) { - out.write(separator); - } - serialize(out, list.get(i), eoi, separators, level+1, - nullSequence, escaped, escapeChar, needsEscape); + case PRIMITIVE: { + LazyUtils.writePrimitiveUTF8(out, obj, + (PrimitiveObjectInspector) objInspector, escaped, escapeChar, + needsEscape); + return; + } + case LIST: { + char separator = (char) separators[level]; + ListObjectInspector loi = (ListObjectInspector) objInspector; + List list = loi.getList(obj); + ObjectInspector eoi = loi.getListElementObjectInspector(); + if (list == null) { + out.write(nullSequence.getBytes(), 0, nullSequence.getLength()); + } else { + for (int i = 0; i < list.size(); i++) { + if (i > 0) { + out.write(separator); } + serialize(out, list.get(i), eoi, separators, level + 1, nullSequence, + escaped, escapeChar, needsEscape); } - return; } - case MAP: { - char separator = (char)separators[level]; - char keyValueSeparator = (char)separators[level+1]; - MapObjectInspector moi = (MapObjectInspector)objInspector; - ObjectInspector koi = moi.getMapKeyObjectInspector(); - ObjectInspector voi = moi.getMapValueObjectInspector(); - - Map map = moi.getMap(obj); - if (map == null) { - out.write(nullSequence.getBytes(), 0, nullSequence.getLength()); - } else { - boolean first = true; - for (Map.Entry entry: map.entrySet()) { - if (first) { - first = false; - } else { - out.write(separator); - } - serialize(out, entry.getKey(), koi, separators, level+2, - nullSequence, escaped, escapeChar, needsEscape); - out.write(keyValueSeparator); - serialize(out, entry.getValue(), voi, separators, level+2, - nullSequence, escaped, escapeChar, needsEscape); + return; + } + case MAP: { + char separator = (char) separators[level]; + char keyValueSeparator = (char) separators[level + 1]; + MapObjectInspector moi = (MapObjectInspector) objInspector; + ObjectInspector koi = moi.getMapKeyObjectInspector(); + ObjectInspector voi = moi.getMapValueObjectInspector(); + + Map map = moi.getMap(obj); + if (map == null) { + out.write(nullSequence.getBytes(), 0, nullSequence.getLength()); + } else { + boolean first = true; + for (Map.Entry entry : map.entrySet()) { + if (first) { + first = false; + } else { + out.write(separator); } + serialize(out, entry.getKey(), koi, separators, level + 2, + nullSequence, escaped, escapeChar, needsEscape); + out.write(keyValueSeparator); + serialize(out, entry.getValue(), voi, separators, level + 2, + nullSequence, escaped, escapeChar, needsEscape); } - return; } - case STRUCT: { - char separator = (char)separators[level]; - StructObjectInspector soi = (StructObjectInspector)objInspector; - List fields = soi.getAllStructFieldRefs(); - List list = soi.getStructFieldsDataAsList(obj); - if (list == null) { - out.write(nullSequence.getBytes(), 0, nullSequence.getLength()); - } else { - for (int i=0; i0) { - out.write(separator); - } - serialize(out, list.get(i), - fields.get(i).getFieldObjectInspector(), separators, level+1, - nullSequence, escaped, escapeChar, needsEscape); + return; + } + case STRUCT: { + char separator = (char) separators[level]; + StructObjectInspector soi = (StructObjectInspector) objInspector; + List fields = soi.getAllStructFieldRefs(); + List list = soi.getStructFieldsDataAsList(obj); + if (list == null) { + out.write(nullSequence.getBytes(), 0, nullSequence.getLength()); + } else { + for (int i = 0; i < list.size(); i++) { + if (i > 0) { + out.write(separator); } + serialize(out, list.get(i), fields.get(i).getFieldObjectInspector(), + separators, level + 1, nullSequence, escaped, escapeChar, + needsEscape); } - return; } + return; } - + } + throw new RuntimeException("Unknown category type: " + objInspector.getCategory()); } @@ -526,7 +554,8 @@ } /** - * @param useJSONSerialize the useJSONSerialize to set + * @param useJSONSerialize + * the useJSONSerialize to set */ public void setUseJSONSerialize(boolean useJSONSerialize) { this.useJSONSerialize = useJSONSerialize; Index: serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazyObjectInspectorFactory.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazyObjectInspectorFactory.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazyObjectInspectorFactory.java (working copy) @@ -21,37 +21,30 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; -import java.util.Map; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.UnionStructObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableBooleanObjectInspector; import org.apache.hadoop.io.Text; - /** - * ObjectInspectorFactory is the primary way to create new ObjectInspector instances. + * ObjectInspectorFactory is the primary way to create new ObjectInspector + * instances. * - * SerDe classes should call the static functions in this library to create an ObjectInspector - * to return to the caller of SerDe2.getObjectInspector(). + * SerDe classes should call the static functions in this library to create an + * ObjectInspector to return to the caller of SerDe2.getObjectInspector(). * - * The reason of having caches here is that ObjectInspectors do not have an internal - * state - so ObjectInspectors with the same construction parameters should - * result in exactly the same ObjectInspector. + * The reason of having caches here is that ObjectInspectors do not have an + * internal state - so ObjectInspectors with the same construction parameters + * should result in exactly the same ObjectInspector. */ public class LazyObjectInspectorFactory { + static HashMap, LazySimpleStructObjectInspector> cachedLazySimpleStructObjectInspector = new HashMap, LazySimpleStructObjectInspector>(); - - static HashMap, LazySimpleStructObjectInspector> cachedLazySimpleStructObjectInspector = - new HashMap, LazySimpleStructObjectInspector>(); - public static LazySimpleStructObjectInspector getLazySimpleStructObjectInspector(List structFieldNames, - List structFieldObjectInspectors, byte separator, Text nullSequence, - boolean lastColumnTakesRest, boolean escaped, byte escapeChar) { + public static LazySimpleStructObjectInspector getLazySimpleStructObjectInspector( + List structFieldNames, + List structFieldObjectInspectors, byte separator, + Text nullSequence, boolean lastColumnTakesRest, boolean escaped, + byte escapeChar) { ArrayList signature = new ArrayList(); signature.add(structFieldNames); signature.add(structFieldObjectInspectors); @@ -60,40 +53,44 @@ signature.add(Boolean.valueOf(lastColumnTakesRest)); signature.add(Boolean.valueOf(escaped)); signature.add(Byte.valueOf(escapeChar)); - LazySimpleStructObjectInspector result = cachedLazySimpleStructObjectInspector.get(signature); + LazySimpleStructObjectInspector result = cachedLazySimpleStructObjectInspector + .get(signature); if (result == null) { - result = new LazySimpleStructObjectInspector(structFieldNames, structFieldObjectInspectors, - separator, nullSequence, lastColumnTakesRest, escaped, escapeChar); + result = new LazySimpleStructObjectInspector(structFieldNames, + structFieldObjectInspectors, separator, nullSequence, + lastColumnTakesRest, escaped, escapeChar); cachedLazySimpleStructObjectInspector.put(signature, result); } return result; } - static HashMap, LazyListObjectInspector> cachedLazySimpleListObjectInspector = - new HashMap, LazyListObjectInspector>(); - public static LazyListObjectInspector getLazySimpleListObjectInspector( - ObjectInspector listElementObjectInspector, byte separator, Text nullSequence, - boolean escaped, byte escapeChar) { + static HashMap, LazyListObjectInspector> cachedLazySimpleListObjectInspector = new HashMap, LazyListObjectInspector>(); + + public static LazyListObjectInspector getLazySimpleListObjectInspector( + ObjectInspector listElementObjectInspector, byte separator, + Text nullSequence, boolean escaped, byte escapeChar) { ArrayList signature = new ArrayList(); signature.add(listElementObjectInspector); signature.add(Byte.valueOf(separator)); signature.add(nullSequence.toString()); signature.add(Boolean.valueOf(escaped)); signature.add(Byte.valueOf(escapeChar)); - LazyListObjectInspector result = cachedLazySimpleListObjectInspector.get(signature); + LazyListObjectInspector result = cachedLazySimpleListObjectInspector + .get(signature); if (result == null) { - result = new LazyListObjectInspector(listElementObjectInspector, + result = new LazyListObjectInspector(listElementObjectInspector, separator, nullSequence, escaped, escapeChar); cachedLazySimpleListObjectInspector.put(signature, result); } return result; } - - static HashMap, LazyMapObjectInspector> cachedLazySimpleMapObjectInspector = - new HashMap, LazyMapObjectInspector>(); - public static LazyMapObjectInspector getLazySimpleMapObjectInspector( - ObjectInspector mapKeyObjectInspector, ObjectInspector mapValueObjectInspector, - byte itemSeparator, byte keyValueSeparator, Text nullSequence, boolean escaped, + + static HashMap, LazyMapObjectInspector> cachedLazySimpleMapObjectInspector = new HashMap, LazyMapObjectInspector>(); + + public static LazyMapObjectInspector getLazySimpleMapObjectInspector( + ObjectInspector mapKeyObjectInspector, + ObjectInspector mapValueObjectInspector, byte itemSeparator, + byte keyValueSeparator, Text nullSequence, boolean escaped, byte escapeChar) { ArrayList signature = new ArrayList(); signature.add(mapKeyObjectInspector); @@ -103,14 +100,15 @@ signature.add(nullSequence.toString()); signature.add(Boolean.valueOf(escaped)); signature.add(Byte.valueOf(escapeChar)); - LazyMapObjectInspector result = cachedLazySimpleMapObjectInspector.get(signature); + LazyMapObjectInspector result = cachedLazySimpleMapObjectInspector + .get(signature); if (result == null) { result = new LazyMapObjectInspector(mapKeyObjectInspector, - mapValueObjectInspector, itemSeparator, keyValueSeparator, nullSequence, - escaped, escapeChar); + mapValueObjectInspector, itemSeparator, keyValueSeparator, + nullSequence, escaped, escapeChar); cachedLazySimpleMapObjectInspector.put(signature, result); } return result; } - + } Index: serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyVoidObjectInspector.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyVoidObjectInspector.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyVoidObjectInspector.java (working copy) @@ -21,17 +21,17 @@ import org.apache.hadoop.hive.serde2.objectinspector.primitive.VoidObjectInspector; import org.apache.hadoop.io.NullWritable; - /** * A WritableVoidObjectInspector inspects a NullWritable Object. */ -public class LazyVoidObjectInspector extends AbstractPrimitiveLazyObjectInspector -implements VoidObjectInspector{ +public class LazyVoidObjectInspector extends + AbstractPrimitiveLazyObjectInspector implements + VoidObjectInspector { LazyVoidObjectInspector() { super(PrimitiveObjectInspectorUtils.voidTypeEntry); } - + @Override public Object copyObject(Object o) { return o; Index: serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyByteObjectInspector.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyByteObjectInspector.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyByteObjectInspector.java (working copy) @@ -21,19 +21,18 @@ import org.apache.hadoop.hive.serde2.lazy.LazyByte; import org.apache.hadoop.hive.serde2.objectinspector.primitive.ByteObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; -import org.apache.hadoop.io.BooleanWritable; - /** * A WritableByteObjectInspector inspects a ByteWritable Object. */ -public class LazyByteObjectInspector extends AbstractPrimitiveLazyObjectInspector -implements ByteObjectInspector{ +public class LazyByteObjectInspector extends + AbstractPrimitiveLazyObjectInspector implements + ByteObjectInspector { LazyByteObjectInspector() { super(PrimitiveObjectInspectorUtils.byteTypeEntry); } - + @Override public byte get(Object o) { return getPrimitiveWritableObject(o).get(); @@ -41,7 +40,7 @@ @Override public Object copyObject(Object o) { - return o == null ? null : new LazyByte((LazyByte)o); + return o == null ? null : new LazyByte((LazyByte) o); } @Override Index: serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyFloatObjectInspector.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyFloatObjectInspector.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyFloatObjectInspector.java (working copy) @@ -25,13 +25,14 @@ /** * A FloatObjectInspector inspects a FloatWritable Object. */ -public class LazyFloatObjectInspector extends AbstractPrimitiveLazyObjectInspector -implements FloatObjectInspector{ +public class LazyFloatObjectInspector extends + AbstractPrimitiveLazyObjectInspector implements + FloatObjectInspector { LazyFloatObjectInspector() { super(PrimitiveObjectInspectorUtils.floatTypeEntry); } - + @Override public float get(Object o) { return getPrimitiveWritableObject(o).get(); @@ -39,7 +40,7 @@ @Override public Object copyObject(Object o) { - return o == null ? null : new LazyFloat((LazyFloat)o); + return o == null ? null : new LazyFloat((LazyFloat) o); } @Override Index: serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyStringObjectInspector.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyStringObjectInspector.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyStringObjectInspector.java (working copy) @@ -22,16 +22,15 @@ import org.apache.hadoop.hive.serde2.objectinspector.primitive.StringObjectInspector; import org.apache.hadoop.io.Text; - /** * A WritableStringObjectInspector inspects a Text Object. */ -public class LazyStringObjectInspector extends AbstractPrimitiveLazyObjectInspector -implements StringObjectInspector{ +public class LazyStringObjectInspector extends + AbstractPrimitiveLazyObjectInspector implements StringObjectInspector { boolean escaped; byte escapeChar; - + LazyStringObjectInspector(boolean escaped, byte escapeChar) { super(PrimitiveObjectInspectorUtils.stringTypeEntry); this.escaped = escaped; @@ -40,25 +39,25 @@ @Override public Object copyObject(Object o) { - return o == null ? null : new LazyString((LazyString)o); + return o == null ? null : new LazyString((LazyString) o); } @Override public Text getPrimitiveWritableObject(Object o) { - return o == null ? null : ((LazyString)o).getWritableObject(); + return o == null ? null : ((LazyString) o).getWritableObject(); } - + @Override public String getPrimitiveJavaObject(Object o) { - return o == null ? null : ((LazyString)o).getWritableObject().toString(); + return o == null ? null : ((LazyString) o).getWritableObject().toString(); } public boolean isEscaped() { return escaped; } + public byte getEscapeChar() { return escapeChar; } - - + } Index: serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyDoubleObjectInspector.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyDoubleObjectInspector.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyDoubleObjectInspector.java (working copy) @@ -22,17 +22,17 @@ import org.apache.hadoop.hive.serde2.objectinspector.primitive.DoubleObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; - /** * A WritableDoubleObjectInspector inspects a DoubleWritable Object. */ -public class LazyDoubleObjectInspector extends AbstractPrimitiveLazyObjectInspector -implements DoubleObjectInspector{ +public class LazyDoubleObjectInspector extends + AbstractPrimitiveLazyObjectInspector implements + DoubleObjectInspector { LazyDoubleObjectInspector() { super(PrimitiveObjectInspectorUtils.doubleTypeEntry); } - + @Override public double get(Object o) { return getPrimitiveWritableObject(o).get(); @@ -40,7 +40,7 @@ @Override public Object copyObject(Object o) { - return o == null ? null : new LazyDouble((LazyDouble)o); + return o == null ? null : new LazyDouble((LazyDouble) o); } @Override Index: serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyIntObjectInspector.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyIntObjectInspector.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyIntObjectInspector.java (working copy) @@ -22,17 +22,17 @@ import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; import org.apache.hadoop.io.IntWritable; - /** * A WritableIntObjectInspector inspects a IntWritable Object. */ -public class LazyIntObjectInspector extends AbstractPrimitiveLazyObjectInspector -implements IntObjectInspector{ +public class LazyIntObjectInspector extends + AbstractPrimitiveLazyObjectInspector implements + IntObjectInspector { LazyIntObjectInspector() { super(PrimitiveObjectInspectorUtils.intTypeEntry); } - + @Override public int get(Object o) { return getPrimitiveWritableObject(o).get(); @@ -40,7 +40,7 @@ @Override public Object copyObject(Object o) { - return o == null ? null : new LazyInteger((LazyInteger)o); + return o == null ? null : new LazyInteger((LazyInteger) o); } @Override Index: serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/AbstractPrimitiveLazyObjectInspector.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/AbstractPrimitiveLazyObjectInspector.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/AbstractPrimitiveLazyObjectInspector.java (working copy) @@ -23,22 +23,23 @@ import org.apache.hadoop.io.Writable; /** - * An AbstractPrimitiveLazyObjectInspector for a LazyPrimitive object. + * An AbstractPrimitiveLazyObjectInspector for a LazyPrimitive object. */ -public abstract class AbstractPrimitiveLazyObjectInspector extends AbstractPrimitiveObjectInspector { +public abstract class AbstractPrimitiveLazyObjectInspector + extends AbstractPrimitiveObjectInspector { protected AbstractPrimitiveLazyObjectInspector(PrimitiveTypeEntry typeEntry) { - super(typeEntry); + super(typeEntry); } - + @Override public T getPrimitiveWritableObject(Object o) { - return o == null ? null : ((LazyPrimitive)o).getWritableObject(); + return o == null ? null : ((LazyPrimitive) o).getWritableObject(); } @Override public boolean preferWritable() { return true; } - + } Index: serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyBooleanObjectInspector.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyBooleanObjectInspector.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyBooleanObjectInspector.java (working copy) @@ -22,17 +22,17 @@ import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; import org.apache.hadoop.io.BooleanWritable; - /** * A WritableBooleanObjectInspector inspects a BooleanWritable Object. */ -public class LazyBooleanObjectInspector extends AbstractPrimitiveLazyObjectInspector -implements BooleanObjectInspector{ +public class LazyBooleanObjectInspector extends + AbstractPrimitiveLazyObjectInspector implements + BooleanObjectInspector { LazyBooleanObjectInspector() { super(PrimitiveObjectInspectorUtils.booleanTypeEntry); } - + @Override public boolean get(Object o) { return getPrimitiveWritableObject(o).get(); @@ -40,7 +40,7 @@ @Override public Object copyObject(Object o) { - return o == null ? null : new LazyBoolean((LazyBoolean)o); + return o == null ? null : new LazyBoolean((LazyBoolean) o); } @Override Index: serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyLongObjectInspector.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyLongObjectInspector.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyLongObjectInspector.java (working copy) @@ -22,17 +22,17 @@ import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; import org.apache.hadoop.io.LongWritable; - /** * A WritableLongObjectInspector inspects a LongWritable Object. */ -public class LazyLongObjectInspector extends AbstractPrimitiveLazyObjectInspector -implements LongObjectInspector{ +public class LazyLongObjectInspector extends + AbstractPrimitiveLazyObjectInspector implements + LongObjectInspector { LazyLongObjectInspector() { super(PrimitiveObjectInspectorUtils.longTypeEntry); } - + @Override public long get(Object o) { return getPrimitiveWritableObject(o).get(); @@ -40,7 +40,7 @@ @Override public Object copyObject(Object o) { - return o == null ? null : new LazyLong((LazyLong)o); + return o == null ? null : new LazyLong((LazyLong) o); } @Override Index: serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyShortObjectInspector.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyShortObjectInspector.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyShortObjectInspector.java (working copy) @@ -22,17 +22,17 @@ import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; import org.apache.hadoop.hive.serde2.objectinspector.primitive.ShortObjectInspector; - /** * A WritableShortObjectInspector inspects a ShortWritable Object. */ -public class LazyShortObjectInspector extends AbstractPrimitiveLazyObjectInspector -implements ShortObjectInspector{ +public class LazyShortObjectInspector extends + AbstractPrimitiveLazyObjectInspector implements + ShortObjectInspector { LazyShortObjectInspector() { super(PrimitiveObjectInspectorUtils.shortTypeEntry); } - + @Override public short get(Object o) { return getPrimitiveWritableObject(o).get(); @@ -40,7 +40,7 @@ @Override public Object copyObject(Object o) { - return o == null ? null : new LazyShort((LazyShort)o); + return o == null ? null : new LazyShort((LazyShort) o); } @Override Index: serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyPrimitiveObjectInspectorFactory.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyPrimitiveObjectInspectorFactory.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyPrimitiveObjectInspectorFactory.java (working copy) @@ -20,37 +20,20 @@ import java.util.ArrayList; import java.util.HashMap; -import java.util.List; -import java.util.Map; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.UnionStructObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.AbstractPrimitiveJavaObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableBooleanObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableByteObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableDoubleObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableFloatObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableIntObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableLongObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableShortObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableStringObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableVoidObjectInspector; -import org.apache.hadoop.io.Text; - /** - * LazyPrimitiveObjectInspectorFactory is the primary way to create new ObjectInspector instances. + * LazyPrimitiveObjectInspectorFactory is the primary way to create new + * ObjectInspector instances. * - * SerDe classes should call the static functions in this library to create an ObjectInspector - * to return to the caller of SerDe2.getObjectInspector(). + * SerDe classes should call the static functions in this library to create an + * ObjectInspector to return to the caller of SerDe2.getObjectInspector(). * - * The reason of having caches here is that ObjectInspector is because ObjectInspectors do - * not have an internal state - so ObjectInspectors with the same construction parameters should - * result in exactly the same ObjectInspector. + * The reason of having caches here is that ObjectInspector is because + * ObjectInspectors do not have an internal state - so ObjectInspectors with the + * same construction parameters should result in exactly the same + * ObjectInspector. */ public class LazyPrimitiveObjectInspectorFactory { @@ -62,38 +45,48 @@ public final static LazyFloatObjectInspector lazyFloatObjectInspector = new LazyFloatObjectInspector(); public final static LazyDoubleObjectInspector lazyDoubleObjectInspector = new LazyDoubleObjectInspector(); public final static LazyVoidObjectInspector lazyVoidObjectInspector = new LazyVoidObjectInspector(); - - static HashMap, LazyStringObjectInspector> cachedLazyStringObjectInspector = - new HashMap, LazyStringObjectInspector>(); - public static LazyStringObjectInspector getLazyStringObjectInspector(boolean escaped, byte escapeChar) { + + static HashMap, LazyStringObjectInspector> cachedLazyStringObjectInspector = new HashMap, LazyStringObjectInspector>(); + + public static LazyStringObjectInspector getLazyStringObjectInspector( + boolean escaped, byte escapeChar) { ArrayList signature = new ArrayList(); signature.add(Boolean.valueOf(escaped)); signature.add(Byte.valueOf(escapeChar)); - LazyStringObjectInspector result = cachedLazyStringObjectInspector.get(signature); + LazyStringObjectInspector result = cachedLazyStringObjectInspector + .get(signature); if (result == null) { result = new LazyStringObjectInspector(escaped, escapeChar); cachedLazyStringObjectInspector.put(signature, result); } return result; } - + public static AbstractPrimitiveLazyObjectInspector getLazyObjectInspector( PrimitiveCategory primitiveCategory, boolean escaped, byte escapeChar) { - - switch(primitiveCategory) { - case BOOLEAN: return lazyBooleanObjectInspector; - case BYTE: return lazyByteObjectInspector; - case SHORT: return lazyShortObjectInspector; - case INT: return lazyIntObjectInspector; - case LONG: return lazyLongObjectInspector; - case FLOAT: return lazyFloatObjectInspector; - case DOUBLE: return lazyDoubleObjectInspector; - case STRING: return getLazyStringObjectInspector(escaped, escapeChar); + + switch (primitiveCategory) { + case BOOLEAN: + return lazyBooleanObjectInspector; + case BYTE: + return lazyByteObjectInspector; + case SHORT: + return lazyShortObjectInspector; + case INT: + return lazyIntObjectInspector; + case LONG: + return lazyLongObjectInspector; + case FLOAT: + return lazyFloatObjectInspector; + case DOUBLE: + return lazyDoubleObjectInspector; + case STRING: + return getLazyStringObjectInspector(escaped, escapeChar); case VOID: - default: - throw new RuntimeException("Internal error: Cannot find ObjectInspector " - + " for " + primitiveCategory); + default: + throw new RuntimeException("Internal error: Cannot find ObjectInspector " + + " for " + primitiveCategory); } } - + } Index: serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazyListObjectInspector.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazyListObjectInspector.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazyListObjectInspector.java (working copy) @@ -30,23 +30,25 @@ /** * LazyListObjectInspector works on array data that is stored in LazyArray. * - * Always use the ObjectInspectorFactory to create new ObjectInspector objects, instead - * of directly creating an instance of this class. + * Always use the ObjectInspectorFactory to create new ObjectInspector objects, + * instead of directly creating an instance of this class. */ public class LazyListObjectInspector implements ListObjectInspector { - public static final Log LOG = LogFactory.getLog(LazyListObjectInspector.class.getName()); - + public static final Log LOG = LogFactory.getLog(LazyListObjectInspector.class + .getName()); + ObjectInspector listElementObjectInspector; - + byte separator; Text nullSequence; boolean escaped; byte escapeChar; - - /** Call ObjectInspectorFactory.getLazySimpleListObjectInspector instead. + + /** + * Call ObjectInspectorFactory.getLazySimpleListObjectInspector instead. */ - protected LazyListObjectInspector(ObjectInspector listElementObjectInspector, + protected LazyListObjectInspector(ObjectInspector listElementObjectInspector, byte separator, Text nullSequence, boolean escaped, byte escapeChar) { this.listElementObjectInspector = listElementObjectInspector; this.separator = separator; @@ -65,7 +67,7 @@ public ObjectInspector getListElementObjectInspector() { return listElementObjectInspector; } - + // with data @Override public Object getListElement(Object data, int index) { @@ -75,7 +77,7 @@ LazyArray array = (LazyArray) data; return array.getListElementObject(index); } - + @Override public int getListLength(Object data) { if (data == null) { @@ -84,7 +86,7 @@ LazyArray array = (LazyArray) data; return array.getListLength(); } - + @Override public List getList(Object data) { if (data == null) { @@ -96,28 +98,29 @@ @Override public String getTypeName() { - return org.apache.hadoop.hive.serde.Constants.LIST_TYPE_NAME - + "<" + listElementObjectInspector.getTypeName() + ">"; + return org.apache.hadoop.hive.serde.Constants.LIST_TYPE_NAME + "<" + + listElementObjectInspector.getTypeName() + ">"; } /** - * Returns the separator for delimiting items in this array. - * Called by LazyArray.init(...). + * Returns the separator for delimiting items in this array. Called by + * LazyArray.init(...). */ public byte getSeparator() { return separator; } /** - * Returns the NullSequence for this array. - * Called by LazyArray.init(...). + * Returns the NullSequence for this array. Called by LazyArray.init(...). */ public Text getNullSequence() { return nullSequence; } + public boolean isEscaped() { return escaped; } + public byte getEscapeChar() { return escapeChar; } Index: serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazyMapObjectInspector.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazyMapObjectInspector.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazyMapObjectInspector.java (working copy) @@ -30,28 +30,30 @@ /** * LazyMapObjectInspector works on struct data that is stored in LazyStruct. * - * Always use the ObjectInspectorFactory to create new ObjectInspector objects, instead - * of directly creating an instance of this class. + * Always use the ObjectInspectorFactory to create new ObjectInspector objects, + * instead of directly creating an instance of this class. */ public class LazyMapObjectInspector implements MapObjectInspector { - public static final Log LOG = LogFactory.getLog(LazyMapObjectInspector.class.getName()); - + public static final Log LOG = LogFactory.getLog(LazyMapObjectInspector.class + .getName()); + ObjectInspector mapKeyObjectInspector; ObjectInspector mapValueObjectInspector; - + byte itemSeparator; - byte keyValueSeparator; + byte keyValueSeparator; Text nullSequence; boolean escaped; byte escapeChar; - - /** Call ObjectInspectorFactory.getStandardListObjectInspector instead. + + /** + * Call ObjectInspectorFactory.getStandardListObjectInspector instead. */ protected LazyMapObjectInspector(ObjectInspector mapKeyObjectInspector, - ObjectInspector mapValueObjectInspector, - byte itemSeparator, byte keyValueSeparator, Text nullSequence, - boolean escaped, byte escapeChar) { + ObjectInspector mapValueObjectInspector, byte itemSeparator, + byte keyValueSeparator, Text nullSequence, boolean escaped, + byte escapeChar) { this.mapKeyObjectInspector = mapKeyObjectInspector; this.mapValueObjectInspector = mapValueObjectInspector; @@ -69,8 +71,8 @@ @Override public String getTypeName() { - return org.apache.hadoop.hive.serde.Constants.MAP_TYPE_NAME - + "<" + mapKeyObjectInspector.getTypeName() + "," + return org.apache.hadoop.hive.serde.Constants.MAP_TYPE_NAME + "<" + + mapKeyObjectInspector.getTypeName() + "," + mapValueObjectInspector.getTypeName() + ">"; } @@ -89,7 +91,7 @@ if (data == null) { return null; } - return ((LazyMap)data).getMapValueElement(key); + return ((LazyMap) data).getMapValueElement(key); } @Override @@ -97,7 +99,7 @@ if (data == null) { return null; } - return ((LazyMap)data).getMap(); + return ((LazyMap) data).getMap(); } @Override @@ -105,22 +107,26 @@ if (data == null) { return -1; } - return ((LazyMap)data).getMapSize(); + return ((LazyMap) data).getMapSize(); } - + // Called by LazyMap public byte getItemSeparator() { return itemSeparator; } + public byte getKeyValueSeparator() { return keyValueSeparator; } + public Text getNullSequence() { return nullSequence; } + public boolean isEscaped() { return escaped; } + public byte getEscapeChar() { return escapeChar; } Index: serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazySimpleStructObjectInspector.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazySimpleStructObjectInspector.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazySimpleStructObjectInspector.java (working copy) @@ -31,24 +31,27 @@ import org.apache.hadoop.io.Text; /** - * LazySimpleStructObjectInspector works on struct data that is stored in LazyStruct. + * LazySimpleStructObjectInspector works on struct data that is stored in + * LazyStruct. * - * The names of the struct fields and the internal structure of the struct fields - * are specified in the ctor of the LazySimpleStructObjectInspector. + * The names of the struct fields and the internal structure of the struct + * fields are specified in the ctor of the LazySimpleStructObjectInspector. * - * Always use the ObjectInspectorFactory to create new ObjectInspector objects, instead - * of directly creating an instance of this class. + * Always use the ObjectInspectorFactory to create new ObjectInspector objects, + * instead of directly creating an instance of this class. */ public class LazySimpleStructObjectInspector extends StructObjectInspector { - public static final Log LOG = LogFactory.getLog(LazySimpleStructObjectInspector.class.getName()); - + public static final Log LOG = LogFactory + .getLog(LazySimpleStructObjectInspector.class.getName()); + protected static class MyField implements StructField { protected int fieldID; protected String fieldName; protected ObjectInspector fieldObjectInspector; - - public MyField(int fieldID, String fieldName, ObjectInspector fieldObjectInspector) { + + public MyField(int fieldID, String fieldName, + ObjectInspector fieldObjectInspector) { this.fieldID = fieldID; this.fieldName = fieldName.toLowerCase(); this.fieldObjectInspector = fieldObjectInspector; @@ -57,70 +60,81 @@ public int getFieldID() { return fieldID; } + public String getFieldName() { return fieldName; } + public ObjectInspector getFieldObjectInspector() { return fieldObjectInspector; } - + + @Override public String toString() { return "" + fieldID + ":" + fieldName; } } - + protected List fields; - + @Override public String getTypeName() { return ObjectInspectorUtils.getStandardStructTypeName(this); } - - + byte separator; - Text nullSequence; + Text nullSequence; boolean lastColumnTakesRest; boolean escaped; byte escapeChar; - - /** Call ObjectInspectorFactory.getLazySimpleStructObjectInspector instead. + + /** + * Call ObjectInspectorFactory.getLazySimpleStructObjectInspector instead. */ - protected LazySimpleStructObjectInspector(List structFieldNames, List structFieldObjectInspectors, - byte separator, Text nullSequence, boolean lastColumnTakesRest, - boolean escaped, byte escapeChar) { - init(structFieldNames, structFieldObjectInspectors, separator, nullSequence, lastColumnTakesRest, - escaped, escapeChar); + protected LazySimpleStructObjectInspector(List structFieldNames, + List structFieldObjectInspectors, byte separator, + Text nullSequence, boolean lastColumnTakesRest, boolean escaped, + byte escapeChar) { + init(structFieldNames, structFieldObjectInspectors, separator, + nullSequence, lastColumnTakesRest, escaped, escapeChar); } - protected void init(List structFieldNames, List structFieldObjectInspectors, - byte separator, Text nullSequence, boolean lastColumnTakesRest, boolean escaped, byte escapeChar) { - assert(structFieldNames.size() == structFieldObjectInspectors.size()); - + + protected void init(List structFieldNames, + List structFieldObjectInspectors, byte separator, + Text nullSequence, boolean lastColumnTakesRest, boolean escaped, + byte escapeChar) { + assert (structFieldNames.size() == structFieldObjectInspectors.size()); + this.separator = separator; this.nullSequence = nullSequence; this.lastColumnTakesRest = lastColumnTakesRest; this.escaped = escaped; this.escapeChar = escapeChar; - - fields = new ArrayList(structFieldNames.size()); - for(int i=0; i(structFieldNames.size()); + for (int i = 0; i < structFieldNames.size(); i++) { + fields.add(new MyField(i, structFieldNames.get(i), + structFieldObjectInspectors.get(i))); } } - - protected LazySimpleStructObjectInspector(List fields, byte separator, Text nullSequence) { + + protected LazySimpleStructObjectInspector(List fields, + byte separator, Text nullSequence) { init(fields, separator, nullSequence); } - protected void init(List fields, byte separator, Text nullSequence) { + + protected void init(List fields, byte separator, + Text nullSequence) { this.separator = separator; this.nullSequence = nullSequence; - - this.fields = new ArrayList(fields.size()); - for(int i=0; i(fields.size()); + for (int i = 0; i < fields.size(); i++) { + this.fields.add(new MyField(i, fields.get(i).getFieldName(), fields + .get(i).getFieldObjectInspector())); } } - @Override public final Category getCategory() { return Category.STRUCT; @@ -131,6 +145,7 @@ public StructField getStructFieldRef(String fieldName) { return ObjectInspectorUtils.getStandardStructFieldRef(fieldName, fields); } + @Override public List getAllStructFieldRefs() { return fields; @@ -142,12 +157,12 @@ if (data == null) { return null; } - LazyStruct struct = (LazyStruct)data; + LazyStruct struct = (LazyStruct) data; MyField f = (MyField) fieldRef; - + int fieldID = f.getFieldID(); - assert(fieldID >= 0 && fieldID < fields.size()); - + assert (fieldID >= 0 && fieldID < fields.size()); + return struct.getField(fieldID); } @@ -156,7 +171,7 @@ if (data == null) { return null; } - LazyStruct struct = (LazyStruct)data; + LazyStruct struct = (LazyStruct) data; return struct.getFieldsAsList(); } @@ -164,17 +179,21 @@ public byte getSeparator() { return separator; } + public Text getNullSequence() { return nullSequence; } + public boolean getLastColumnTakesRest() { return lastColumnTakesRest; } + public boolean isEscaped() { return escaped; } + public byte getEscapeChar() { return escapeChar; } - + } Index: serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyMap.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyMap.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyMap.java (working copy) @@ -18,7 +18,6 @@ package org.apache.hadoop.hive.serde2.lazy; import java.util.Arrays; -import java.util.HashMap; import java.util.LinkedHashMap; import java.util.Map; @@ -28,59 +27,55 @@ import org.apache.hadoop.io.Text; /** - * LazyMap stores a map of Primitive LazyObjects to LazyObjects. - * Note that the keys of the map cannot contain null. + * LazyMap stores a map of Primitive LazyObjects to LazyObjects. Note that the + * keys of the map cannot contain null. * - * LazyMap does not deal with the case of a NULL map. That is handled - * by the parent LazyObject. + * LazyMap does not deal with the case of a NULL map. That is handled by the + * parent LazyObject. */ public class LazyMap extends LazyNonPrimitive { - + /** * Whether the data is already parsed or not. */ boolean parsed = false; - + /** - * The size of the map. - * Only valid when the data is parsed. - * -1 when the map is NULL. + * The size of the map. Only valid when the data is parsed. -1 when the map is + * NULL. */ int mapSize = 0; - + /** - * The beginning position of key[i]. - * Only valid when the data is parsed. - * Note that keyStart[mapSize] = begin + length + 1; - * that makes sure we can use the same formula to compute the - * length of each value in the map. + * The beginning position of key[i]. Only valid when the data is parsed. Note + * that keyStart[mapSize] = begin + length + 1; that makes sure we can use the + * same formula to compute the length of each value in the map. */ int[] keyStart; - + /** - * The end position of key[i] (the position of the key-value separator). - * Only valid when the data is parsed. - */ + * The end position of key[i] (the position of the key-value separator). Only + * valid when the data is parsed. + */ int[] keyEnd; /** * The keys are stored in an array of LazyPrimitives. */ - LazyPrimitive[] keyObjects; + LazyPrimitive[] keyObjects; /** - * Whether init() is called on keyObjects[i]. + * Whether init() is called on keyObjects[i]. */ boolean[] keyInited; /** - * The values are stored in an array of LazyObjects. - * value[index] will start from KeyEnd[index] + 1, - * and ends before KeyStart[index+1] - 1. + * The values are stored in an array of LazyObjects. value[index] will start + * from KeyEnd[index] + 1, and ends before KeyStart[index+1] - 1. */ LazyObject[] valueObjects; /** * Whether init() is called on valueObjects[i] */ boolean[] valueInited; - + /** * Construct a LazyMap object with the ObjectInspector. */ @@ -90,6 +85,7 @@ /** * Set the row data for this LazyArray. + * * @see LazyObject#init(ByteArrayRef, int, int) */ @Override @@ -97,27 +93,27 @@ super.init(bytes, start, length); parsed = false; } - + /** - * Enlarge the size of arrays storing information for the elements inside - * the array. + * Enlarge the size of arrays storing information for the elements inside the + * array. */ protected void enlargeArrays() { if (keyStart == null) { int initialSize = 2; keyStart = new int[initialSize]; keyEnd = new int[initialSize]; - keyObjects = new LazyPrimitive[initialSize]; + keyObjects = new LazyPrimitive[initialSize]; valueObjects = new LazyObject[initialSize]; keyInited = new boolean[initialSize]; valueInited = new boolean[initialSize]; } else { - keyStart = Arrays.copyOf(keyStart, keyStart.length*2); - keyEnd = Arrays.copyOf(keyEnd, keyEnd.length*2); - keyObjects = Arrays.copyOf(keyObjects, keyObjects.length*2); - valueObjects = Arrays.copyOf(valueObjects, valueObjects.length*2); - keyInited = Arrays.copyOf(keyInited, keyInited.length*2); - valueInited = Arrays.copyOf(valueInited, valueInited.length*2); + keyStart = Arrays.copyOf(keyStart, keyStart.length * 2); + keyEnd = Arrays.copyOf(keyEnd, keyEnd.length * 2); + keyObjects = Arrays.copyOf(keyObjects, keyObjects.length * 2); + valueObjects = Arrays.copyOf(valueObjects, valueObjects.length * 2); + keyInited = Arrays.copyOf(keyInited, keyInited.length * 2); + valueInited = Arrays.copyOf(valueInited, valueInited.length * 2); } } @@ -126,44 +122,44 @@ */ private void parse() { parsed = true; - + byte itemSeparator = oi.getItemSeparator(); - byte keyValueSeparator = oi.getKeyValueSeparator(); + byte keyValueSeparator = oi.getKeyValueSeparator(); boolean isEscaped = oi.isEscaped(); byte escapeChar = oi.getEscapeChar(); - + // empty array? if (length == 0) { mapSize = 0; return; } - + mapSize = 0; int arrayByteEnd = start + length; int elementByteBegin = start; int keyValueSeparatorPosition = -1; int elementByteEnd = start; byte[] bytes = this.bytes.getData(); - + // Go through all bytes in the byte[] while (elementByteEnd <= arrayByteEnd) { // End of entry reached? - if (elementByteEnd == arrayByteEnd + if (elementByteEnd == arrayByteEnd || bytes[elementByteEnd] == itemSeparator) { // Array full? if (keyStart == null || mapSize + 1 == keyStart.length) { enlargeArrays(); } keyStart[mapSize] = elementByteBegin; - // If no keyValueSeparator is seen, all bytes belong to key, and + // If no keyValueSeparator is seen, all bytes belong to key, and // value will be NULL. - keyEnd[mapSize] = (keyValueSeparatorPosition == -1 - ? elementByteEnd: keyValueSeparatorPosition); + keyEnd[mapSize] = (keyValueSeparatorPosition == -1 ? elementByteEnd + : keyValueSeparatorPosition); // reset keyValueSeparatorPosition keyValueSeparatorPosition = -1; mapSize++; elementByteBegin = elementByteEnd + 1; - elementByteEnd ++; + elementByteEnd++; } else { // Is this the first keyValueSeparator in this entry? if (keyValueSeparatorPosition == -1 @@ -171,15 +167,15 @@ keyValueSeparatorPosition = elementByteEnd; } if (isEscaped && bytes[elementByteEnd] == escapeChar - && elementByteEnd+1 < arrayByteEnd) { + && elementByteEnd + 1 < arrayByteEnd) { // ignore the char after escape_char elementByteEnd += 2; } else { - elementByteEnd ++; + elementByteEnd++; } } } - + // This makes sure we can use the same formula to compute the // length of each value in the map. keyStart[mapSize] = arrayByteEnd + 1; @@ -189,19 +185,20 @@ Arrays.fill(valueInited, 0, mapSize, false); } } - + /** * Get the value in the map for the key. * - * If there are multiple matches (which is possible in the serialized - * format), only the first one is returned. + * If there are multiple matches (which is possible in the serialized format), + * only the first one is returned. * - * The most efficient way to get the value for the key is to serialize the - * key and then try to find it in the array. We do linear search because in - * most cases, user only wants to get one or two values out of the map, and - * the cost of building up a HashMap is substantially higher. + * The most efficient way to get the value for the key is to serialize the key + * and then try to find it in the array. We do linear search because in most + * cases, user only wants to get one or two values out of the map, and the + * cost of building up a HashMap is substantially higher. * - * @param key The key object that we are looking for. + * @param key + * The key object that we are looking for. * @return The corresponding value object, or NULL if not found */ public Object getMapValueElement(Object key) { @@ -209,83 +206,94 @@ parse(); } // search for the key - for (int i=0; i lazyKeyI = uncheckedGetKey(i); - if (lazyKeyI == null) continue; - // getWritableObject() will convert LazyPrimitive to actual primitive writable objects. + for (int i = 0; i < mapSize; i++) { + LazyPrimitive lazyKeyI = uncheckedGetKey(i); + if (lazyKeyI == null) { + continue; + } + // getWritableObject() will convert LazyPrimitive to actual primitive + // writable objects. Object keyI = lazyKeyI.getWritableObject(); - if (keyI == null) continue; + if (keyI == null) { + continue; + } if (keyI.equals(key)) { // Got a match, return the value LazyObject v = uncheckedGetValue(i); return v == null ? v : v.getObject(); } } - + return null; } /** * Get the value object with the index without checking parsed. - * @param index The index into the array starting from 0 + * + * @param index + * The index into the array starting from 0 */ private LazyObject uncheckedGetValue(int index) { Text nullSequence = oi.getNullSequence(); int valueIBegin = keyEnd[index] + 1; - int valueILength = keyStart[index+1] - 1 - valueIBegin; - if (valueILength < 0 || - ((valueILength == nullSequence.getLength()) - && 0 == LazyUtils.compare(bytes.getData(), valueIBegin, valueILength, - nullSequence.getBytes(), 0, nullSequence.getLength()))) { - return null; + int valueILength = keyStart[index + 1] - 1 - valueIBegin; + if (valueILength < 0 + || ((valueILength == nullSequence.getLength()) && 0 == LazyUtils + .compare(bytes.getData(), valueIBegin, valueILength, nullSequence + .getBytes(), 0, nullSequence.getLength()))) { + return null; } if (!valueInited[index]) { valueInited[index] = true; if (valueObjects[index] == null) { - valueObjects[index] = LazyFactory.createLazyObject( - ((MapObjectInspector)oi).getMapValueObjectInspector()); + valueObjects[index] = LazyFactory + .createLazyObject(((MapObjectInspector) oi) + .getMapValueObjectInspector()); } valueObjects[index].init(bytes, valueIBegin, valueILength); } return valueObjects[index]; } - + /** * Get the key object with the index without checking parsed. - * @param index The index into the array starting from 0 + * + * @param index + * The index into the array starting from 0 */ - private LazyPrimitive uncheckedGetKey(int index) { - Text nullSequence = oi.getNullSequence(); + private LazyPrimitive uncheckedGetKey(int index) { + Text nullSequence = oi.getNullSequence(); int keyIBegin = keyStart[index]; int keyILength = keyEnd[index] - keyStart[index]; - if (keyILength < 0 || - ((keyILength == nullSequence.getLength()) - && 0 == LazyUtils.compare(bytes.getData(), keyIBegin, keyILength, - nullSequence.getBytes(), 0, nullSequence.getLength()))) { + if (keyILength < 0 + || ((keyILength == nullSequence.getLength()) && 0 == LazyUtils.compare( + bytes.getData(), keyIBegin, keyILength, nullSequence.getBytes(), 0, + nullSequence.getLength()))) { return null; } if (!keyInited[index]) { keyInited[index] = true; if (keyObjects[index] == null) { // Keys are always primitive - keyObjects[index] = LazyFactory.createLazyPrimitiveClass( - (PrimitiveObjectInspector)((MapObjectInspector)oi).getMapKeyObjectInspector()); + keyObjects[index] = LazyFactory + .createLazyPrimitiveClass((PrimitiveObjectInspector) ((MapObjectInspector) oi) + .getMapKeyObjectInspector()); } keyObjects[index].init(bytes, keyIBegin, keyILength); } return keyObjects[index]; } - + /** - * cachedMap is reused for different calls to getMap(). - * But each LazyMap has a separate cachedMap so we won't overwrite the - * data by accident. + * cachedMap is reused for different calls to getMap(). But each LazyMap has a + * separate cachedMap so we won't overwrite the data by accident. */ LinkedHashMap cachedMap; - + /** - * Return the map object representing this LazyMap. - * Note that the keyObjects will be Writable primitive objects. + * Return the map object representing this LazyMap. Note that the keyObjects + * will be Writable primitive objects. + * * @return the map object */ public Map getMap() { @@ -298,11 +306,13 @@ } else { cachedMap.clear(); } - + // go through each element of the map for (int i = 0; i < mapSize; i++) { - LazyPrimitive lazyKey = uncheckedGetKey(i); - if (lazyKey == null) continue; + LazyPrimitive lazyKey = uncheckedGetKey(i); + if (lazyKey == null) { + continue; + } Object key = lazyKey.getObject(); // do not overwrite if there are duplicate keys if (key != null && !cachedMap.containsKey(key)) { @@ -316,7 +326,8 @@ /** * Get the size of the map represented by this LazyMap. - * @return The size of the map, -1 for NULL map. + * + * @return The size of the map, -1 for NULL map. */ public int getMapSize() { if (!parsed) { Index: serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyArray.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyArray.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyArray.java (working copy) @@ -28,8 +28,8 @@ /** * LazyArray stores an array of Lazy Objects. * - * LazyArray does not deal with the case of a NULL array. That is handled - * by the parent LazyObject. + * LazyArray does not deal with the case of a NULL array. That is handled by the + * parent LazyObject. */ public class LazyArray extends LazyNonPrimitive { @@ -38,36 +38,36 @@ */ boolean parsed = false; /** - * The length of the array. - * Only valid when the data is parsed. - * -1 when the array is NULL. + * The length of the array. Only valid when the data is parsed. -1 when the + * array is NULL. */ int arrayLength = 0; - + /** - * The start positions of array elements. - * Only valid when the data is parsed. - * Note that startPosition[arrayLength] = begin + length + 1; - * that makes sure we can use the same formula to compute the - * length of each element of the array. + * The start positions of array elements. Only valid when the data is parsed. + * Note that startPosition[arrayLength] = begin + length + 1; that makes sure + * we can use the same formula to compute the length of each element of the + * array. */ int[] startPosition; - + /** * Whether init() has been called on the element or not. */ boolean[] elementInited; - + /** - * The elements of the array. Note that we do arrayElements[i]. - * init(bytes, begin, length) only when that element is accessed. + * The elements of the array. Note that we do arrayElements[i]. init(bytes, + * begin, length) only when that element is accessed. */ LazyObject[] arrayElements; /** * Construct a LazyArray object with the ObjectInspector. - * @param oi the oi representing the type of this LazyArray as well as meta - * information like separator etc. + * + * @param oi + * the oi representing the type of this LazyArray as well as meta + * information like separator etc. */ protected LazyArray(LazyListObjectInspector oi) { super(oi); @@ -75,6 +75,7 @@ /** * Set the row data for this LazyArray. + * * @see LazyObject#init(ByteArrayRef, int, int) */ @Override @@ -82,84 +83,83 @@ super.init(bytes, start, length); parsed = false; } - + /** - * Enlarge the size of arrays storing information for the elements inside - * the array. + * Enlarge the size of arrays storing information for the elements inside the + * array. */ private void enlargeArrays() { if (startPosition == null) { int initialSize = 2; - startPosition = new int[initialSize]; + startPosition = new int[initialSize]; arrayElements = new LazyObject[initialSize]; elementInited = new boolean[initialSize]; } else { - startPosition = Arrays.copyOf(startPosition, startPosition.length*2); - arrayElements = Arrays.copyOf(arrayElements, arrayElements.length*2); - elementInited = Arrays.copyOf(elementInited, elementInited.length*2); + startPosition = Arrays.copyOf(startPosition, startPosition.length * 2); + arrayElements = Arrays.copyOf(arrayElements, arrayElements.length * 2); + elementInited = Arrays.copyOf(elementInited, elementInited.length * 2); } } - + /** * Parse the bytes and fill arrayLength and startPosition. */ private void parse() { parsed = true; - + byte separator = oi.getSeparator(); boolean isEscaped = oi.isEscaped(); byte escapeChar = oi.getEscapeChar(); - + // empty array? if (length == 0) { arrayLength = 0; return; } - + byte[] bytes = this.bytes.getData(); - + arrayLength = 0; int arrayByteEnd = start + length; int elementByteBegin = start; int elementByteEnd = start; - + // Go through all bytes in the byte[] while (elementByteEnd <= arrayByteEnd) { // Reached the end of a field? - if (elementByteEnd == arrayByteEnd - || bytes[elementByteEnd] == separator) { + if (elementByteEnd == arrayByteEnd || bytes[elementByteEnd] == separator) { // Array size not big enough? - if (startPosition == null || arrayLength+1 == startPosition.length) { + if (startPosition == null || arrayLength + 1 == startPosition.length) { enlargeArrays(); } startPosition[arrayLength] = elementByteBegin; arrayLength++; elementByteBegin = elementByteEnd + 1; - elementByteEnd ++; + elementByteEnd++; } else { if (isEscaped && bytes[elementByteEnd] == escapeChar - && elementByteEnd+1 < arrayByteEnd) { + && elementByteEnd + 1 < arrayByteEnd) { // ignore the char after escape_char elementByteEnd += 2; } else { - elementByteEnd ++; + elementByteEnd++; } - } + } } // Store arrayByteEnd+1 in startPosition[arrayLength] // so that we can use the same formula to compute the length of // each element in the array: startPosition[i+1] - startPosition[i] - 1 startPosition[arrayLength] = arrayByteEnd + 1; - + if (arrayLength > 0) { Arrays.fill(elementInited, 0, arrayLength, false); } - + } - + /** - * Returns the actual primitive object at the index position - * inside the array represented by this LazyObject. + * Returns the actual primitive object at the index position inside the array + * represented by this LazyObject. */ public Object getListElementObject(int index) { if (!parsed) { @@ -170,34 +170,35 @@ } return uncheckedGetElement(index); } - + /** * Get the element without checking out-of-bound index. */ private Object uncheckedGetElement(int index) { Text nullSequence = oi.getNullSequence(); - int elementLength = startPosition[index+1] - startPosition[index] - 1; - if (elementLength == nullSequence.getLength() - && 0 == LazyUtils.compare(bytes.getData(), startPosition[index], - elementLength, nullSequence.getBytes(), 0, - nullSequence.getLength())) { + int elementLength = startPosition[index + 1] - startPosition[index] - 1; + if (elementLength == nullSequence.getLength() + && 0 == LazyUtils + .compare(bytes.getData(), startPosition[index], elementLength, + nullSequence.getBytes(), 0, nullSequence.getLength())) { return null; } else { if (!elementInited[index]) { elementInited[index] = true; if (arrayElements[index] == null) { - arrayElements[index] = LazyFactory.createLazyObject( - ((ListObjectInspector)oi).getListElementObjectInspector()); + arrayElements[index] = LazyFactory + .createLazyObject(((ListObjectInspector) oi) + .getListElementObjectInspector()); } - arrayElements[index].init(bytes, startPosition[index], - elementLength); + arrayElements[index].init(bytes, startPosition[index], elementLength); } } return arrayElements[index].getObject(); } - - /** Returns -1 for null array. + + /** + * Returns -1 for null array. */ public int getListLength() { if (!parsed) { @@ -205,14 +206,15 @@ } return arrayLength; } - - /** - * cachedList is reused every time getList is called. - * Different LazyArray instances cannot share the same cachedList. + + /** + * cachedList is reused every time getList is called. Different LazyArray + * instances cannot share the same cachedList. */ ArrayList cachedList; - /** Returns the List of actual primitive objects. - * Returns null for null array. + + /** + * Returns the List of actual primitive objects. Returns null for null array. */ public List getList() { if (!parsed) { @@ -226,7 +228,7 @@ } else { cachedList.clear(); } - for (int index=0; index * */ -public class LazyBoolean extends LazyPrimitive { +public class LazyBoolean extends + LazyPrimitive { public LazyBoolean(LazyBooleanObjectInspector oi) { super(oi); @@ -44,28 +45,26 @@ super(copy); data = new BooleanWritable(copy.data.get()); } - + @Override public void init(ByteArrayRef bytes, int start, int length) { - if (length == 4 - && Character.toUpperCase(bytes.getData()[start]) == 'T' - && Character.toUpperCase(bytes.getData()[start+1]) == 'R' - && Character.toUpperCase(bytes.getData()[start+2]) == 'U' - && Character.toUpperCase(bytes.getData()[start+3]) == 'E') { + if (length == 4 && Character.toUpperCase(bytes.getData()[start]) == 'T' + && Character.toUpperCase(bytes.getData()[start + 1]) == 'R' + && Character.toUpperCase(bytes.getData()[start + 2]) == 'U' + && Character.toUpperCase(bytes.getData()[start + 3]) == 'E') { data.set(true); isNull = false; } else if (length == 5 - && Character.toUpperCase(bytes.getData()[start]) == 'F' - && Character.toUpperCase(bytes.getData()[start+1]) == 'A' - && Character.toUpperCase(bytes.getData()[start+2]) == 'L' - && Character.toUpperCase(bytes.getData()[start+3]) == 'S' - && Character.toUpperCase(bytes.getData()[start+4]) == 'E') { + && Character.toUpperCase(bytes.getData()[start]) == 'F' + && Character.toUpperCase(bytes.getData()[start + 1]) == 'A' + && Character.toUpperCase(bytes.getData()[start + 2]) == 'L' + && Character.toUpperCase(bytes.getData()[start + 3]) == 'S' + && Character.toUpperCase(bytes.getData()[start + 4]) == 'E') { data.set(false); isNull = false; - } else { + } else { isNull = true; } } - } Index: serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyNonPrimitive.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyNonPrimitive.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyNonPrimitive.java (working copy) @@ -18,12 +18,12 @@ package org.apache.hadoop.hive.serde2.lazy; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.io.WritableComparator; /** * LazyPrimitive stores a primitive Object in a LazyObject. */ -public abstract class LazyNonPrimitive extends LazyObject { +public abstract class LazyNonPrimitive extends + LazyObject { protected ByteArrayRef bytes; protected int start; @@ -31,12 +31,14 @@ /** * Create a LazyNonPrimitive object with the specified ObjectInspector. - * @param oi The ObjectInspector would have to have a hierarchy of - * LazyObjectInspectors with the leaf nodes being - * WritableObjectInspectors. It's used both for accessing the - * type hierarchy of the complex object, as well as getting - * meta information (separator, nullSequence, etc) when parsing - * the lazy object. + * + * @param oi + * The ObjectInspector would have to have a hierarchy of + * LazyObjectInspectors with the leaf nodes being + * WritableObjectInspectors. It's used both for accessing the type + * hierarchy of the complex object, as well as getting meta + * information (separator, nullSequence, etc) when parsing the lazy + * object. */ protected LazyNonPrimitive(OI oi) { super(oi); @@ -44,7 +46,7 @@ start = 0; length = 0; } - + @Override public void init(ByteArrayRef bytes, int start, int length) { if (bytes == null) { @@ -59,9 +61,10 @@ @Override public Object getObject() { - return this; + return this; } - + + @Override public int hashCode() { return LazyUtils.hashBytes(bytes.getData(), start, length); } Index: serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyStruct.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyStruct.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyStruct.java (working copy) @@ -28,15 +28,15 @@ import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; import org.apache.hadoop.io.Text; - /** - * LazyObject for storing a struct. - * The field of a struct can be primitive or non-primitive. - * - * LazyStruct does not deal with the case of a NULL struct. That is handled - * by the parent LazyObject. + * LazyObject for storing a struct. The field of a struct can be primitive or + * non-primitive. + * + * LazyStruct does not deal with the case of a NULL struct. That is handled by + * the parent LazyObject. */ -public class LazyStruct extends LazyNonPrimitive { +public class LazyStruct extends + LazyNonPrimitive { private static Log LOG = LogFactory.getLog(LazyStruct.class.getName()); @@ -46,14 +46,13 @@ boolean parsed; /** - * The start positions of struct fields. - * Only valid when the data is parsed. - * Note that startPosition[arrayLength] = begin + length + 1; - * that makes sure we can use the same formula to compute the - * length of each element of the array. + * The start positions of struct fields. Only valid when the data is parsed. + * Note that startPosition[arrayLength] = begin + length + 1; that makes sure + * we can use the same formula to compute the length of each element of the + * array. */ int[] startPosition; - + /** * The fields of the struct. */ @@ -62,18 +61,20 @@ * Whether init() has been called on the field or not. */ boolean[] fieldInited; - + /** * Construct a LazyStruct object with the ObjectInspector. */ public LazyStruct(LazySimpleStructObjectInspector oi) { super(oi); } - + /** * Set the row data for this LazyStruct. + * * @see LazyObject#init(ByteArrayRef, int, int) */ + @Override public void init(ByteArrayRef bytes, int start, int length) { super.init(bytes, start, length); parsed = false; @@ -81,34 +82,37 @@ boolean missingFieldWarned = false; boolean extraFieldWarned = false; + /** * Parse the byte[] and fill each field. */ private void parse() { - + byte separator = oi.getSeparator(); boolean lastColumnTakesRest = oi.getLastColumnTakesRest(); boolean isEscaped = oi.isEscaped(); byte escapeChar = oi.getEscapeChar(); - + if (fields == null) { - List fieldRefs = ((StructObjectInspector)oi).getAllStructFieldRefs(); + List fieldRefs = ((StructObjectInspector) oi) + .getAllStructFieldRefs(); fields = new LazyObject[fieldRefs.size()]; - for (int i = 0 ; i < fields.length; i++) { - fields[i] = LazyFactory.createLazyObject(fieldRefs.get(i).getFieldObjectInspector()); + for (int i = 0; i < fields.length; i++) { + fields[i] = LazyFactory.createLazyObject(fieldRefs.get(i) + .getFieldObjectInspector()); } - fieldInited = new boolean[fields.length]; - // Extra element to make sure we have the same formula to compute the - // length of each element of the array. - startPosition = new int[fields.length+1]; + fieldInited = new boolean[fields.length]; + // Extra element to make sure we have the same formula to compute the + // length of each element of the array. + startPosition = new int[fields.length + 1]; } - + int structByteEnd = start + length; int fieldId = 0; int fieldByteBegin = start; int fieldByteEnd = start; byte[] bytes = this.bytes.getData(); - + // Go through all bytes in the byte[] while (fieldByteEnd <= structByteEnd) { if (fieldByteEnd == structByteEnd || bytes[fieldByteEnd] == separator) { @@ -117,7 +121,7 @@ fieldByteEnd = structByteEnd; } startPosition[fieldId] = fieldByteBegin; - fieldId ++; + fieldId++; if (fieldId == fields.length || fieldByteEnd == structByteEnd) { // All fields have been parsed, or bytes have been parsed. // We need to set the startPosition of fields.length to ensure we @@ -127,14 +131,14 @@ // return these fields as NULLs. for (int i = fieldId; i <= fields.length; i++) { startPosition[i] = fieldByteEnd + 1; - } + } break; } fieldByteBegin = fieldByteEnd + 1; - fieldByteEnd ++; + fieldByteEnd++; } else { if (isEscaped && bytes[fieldByteEnd] == escapeChar - && fieldByteEnd+1 < structByteEnd) { + && fieldByteEnd + 1 < structByteEnd) { // ignore the char after escape_char fieldByteEnd += 2; } else { @@ -142,36 +146,37 @@ } } } - + // Extra bytes at the end? if (!extraFieldWarned && fieldByteEnd < structByteEnd) { extraFieldWarned = true; LOG.warn("Extra bytes detected at the end of the row! Ignoring similar " + "problems."); } - + // Missing fields? if (!missingFieldWarned && fieldId < fields.length) { missingFieldWarned = true; LOG.warn("Missing fields! Expected " + fields.length + " fields but " + "only got " + fieldId + "! Ignoring similar problems."); } - + Arrays.fill(fieldInited, false); - parsed = true; + parsed = true; } - + /** * Get one field out of the struct. * - * If the field is a primitive field, return the actual object. - * Otherwise return the LazyObject. This is because PrimitiveObjectInspector - * does not have control over the object used by the user - the user simply - * directly use the Object instead of going through - * Object PrimitiveObjectInspector.get(Object). + * If the field is a primitive field, return the actual object. Otherwise + * return the LazyObject. This is because PrimitiveObjectInspector does not + * have control over the object used by the user - the user simply directly + * use the Object instead of going through Object + * PrimitiveObjectInspector.get(Object). * - * @param fieldID The field ID - * @return The field as a LazyObject + * @param fieldID + * The field ID + * @return The field as a LazyObject */ public Object getField(int fieldID) { if (!parsed) { @@ -179,24 +184,27 @@ } return uncheckedGetField(fieldID); } - + /** - * Get the field out of the row without checking parsed. - * This is called by both getField and getFieldsAsList. - * @param fieldID The id of the field starting from 0. - * @param nullSequence The sequence representing NULL value. - * @return The value of the field + * Get the field out of the row without checking parsed. This is called by + * both getField and getFieldsAsList. + * + * @param fieldID + * The id of the field starting from 0. + * @param nullSequence + * The sequence representing NULL value. + * @return The value of the field */ private Object uncheckedGetField(int fieldID) { Text nullSequence = oi.getNullSequence(); - // Test the length first so in most cases we avoid doing a byte[] + // Test the length first so in most cases we avoid doing a byte[] // comparison. int fieldByteBegin = startPosition[fieldID]; - int fieldLength = startPosition[fieldID+1] - startPosition[fieldID] - 1; + int fieldLength = startPosition[fieldID + 1] - startPosition[fieldID] - 1; if ((fieldLength < 0) - || (fieldLength == nullSequence.getLength() - && LazyUtils.compare(bytes.getData(), fieldByteBegin, fieldLength, - nullSequence.getBytes(), 0, nullSequence.getLength()) == 0)) { + || (fieldLength == nullSequence.getLength() && LazyUtils.compare(bytes + .getData(), fieldByteBegin, fieldLength, nullSequence.getBytes(), + 0, nullSequence.getLength()) == 0)) { return null; } if (!fieldInited[fieldID]) { @@ -207,8 +215,10 @@ } ArrayList cachedList; + /** * Get the values of the fields as an ArrayList. + * * @return The values of the fields as an ArrayList. */ public ArrayList getFieldsAsList() { @@ -220,12 +230,12 @@ } else { cachedList.clear(); } - for (int i=0; i * */ -public class LazyInteger extends LazyPrimitive { +public class LazyInteger extends + LazyPrimitive { public LazyInteger(LazyIntObjectInspector oi) { super(oi); @@ -46,9 +47,8 @@ public LazyInteger(LazyInteger copy) { super(copy); data = new IntWritable(copy.data.get()); - } - - + } + @Override public void init(ByteArrayRef bytes, int start, int length) { try { @@ -58,7 +58,7 @@ isNull = true; } } - + /** * Parses the string argument as if it was an int value and returns the * result. Throws NumberFormatException if the string does not represent an @@ -67,13 +67,14 @@ * @param bytes * @param start * @param length - * a UTF-8 encoded string representation of an int quantity. + * a UTF-8 encoded string representation of an int quantity. * @return int the value represented by the argument * @exception NumberFormatException - * if the argument could not be parsed as an int quantity. + * if the argument could not be parsed as an int quantity. */ - public static int parseInt(byte[] bytes, int start, int length) throws NumberFormatException { - return parseInt(bytes, start, length, 10); + public static int parseInt(byte[] bytes, int start, int length) + throws NumberFormatException { + return parseInt(bytes, start, length, 10); } /** @@ -85,20 +86,19 @@ * @param bytes * @param start * @param length - * a UTF-8 encoded string representation of an int quantity. + * a UTF-8 encoded string representation of an int quantity. * @param radix - * the base to use for conversion. - * @return the value represented by the argument + * the base to use for conversion. + * @return the value represented by the argument * @exception NumberFormatException - * if the argument could not be parsed as an int quantity. + * if the argument could not be parsed as an int quantity. */ public static int parseInt(byte[] bytes, int start, int length, int radix) - throws NumberFormatException { + throws NumberFormatException { if (bytes == null) { throw new NumberFormatException("String is null"); } - if (radix < Character.MIN_RADIX || - radix > Character.MAX_RADIX) { + if (radix < Character.MIN_RADIX || radix > Character.MAX_RADIX) { throw new NumberFormatException("Invalid radix: " + radix); } if (length == 0) { @@ -109,7 +109,8 @@ if (negative || bytes[start] == '+') { offset++; if (length == 1) { - throw new NumberFormatException(LazyUtils.convertToString(bytes, start, length)); + throw new NumberFormatException(LazyUtils.convertToString(bytes, start, + length)); } } @@ -121,57 +122,63 @@ * @param bytes * @param start * @param length - * a UTF-8 encoded string representation of an int quantity. + * a UTF-8 encoded string representation of an int quantity. * @param radix - * the base to use for conversion. + * the base to use for conversion. * @param offset - * the starting position after the sign (if exists) + * the starting position after the sign (if exists) * @param radix - * the base to use for conversion. + * the base to use for conversion. * @param negative - * whether the number is negative. + * whether the number is negative. * @return the value represented by the argument * @exception NumberFormatException - * if the argument could not be parsed as an int quantity. + * if the argument could not be parsed as an int quantity. */ - private static int parse(byte[] bytes, int start, int length, int offset, int radix, - boolean negative) throws NumberFormatException { - int max = Integer.MIN_VALUE / radix; - int result = 0, end = start + length; - while (offset < end) { - int digit = LazyUtils.digit(bytes[offset++], radix); - if (digit == -1) { - throw new NumberFormatException(LazyUtils.convertToString(bytes, start, length)); - } - if (max > result) { - throw new NumberFormatException(LazyUtils.convertToString(bytes, start, length)); - } - int next = result * radix - digit; - if (next > result) { - throw new NumberFormatException(LazyUtils.convertToString(bytes, start, length)); - } - result = next; + private static int parse(byte[] bytes, int start, int length, int offset, + int radix, boolean negative) throws NumberFormatException { + int max = Integer.MIN_VALUE / radix; + int result = 0, end = start + length; + while (offset < end) { + int digit = LazyUtils.digit(bytes[offset++], radix); + if (digit == -1) { + throw new NumberFormatException(LazyUtils.convertToString(bytes, start, + length)); } - if (!negative) { - result = -result; - if (result < 0) { - throw new NumberFormatException(LazyUtils.convertToString(bytes, start, length)); - } + if (max > result) { + throw new NumberFormatException(LazyUtils.convertToString(bytes, start, + length)); } - return result; + int next = result * radix - digit; + if (next > result) { + throw new NumberFormatException(LazyUtils.convertToString(bytes, start, + length)); + } + result = next; + } + if (!negative) { + result = -result; + if (result < 0) { + throw new NumberFormatException(LazyUtils.convertToString(bytes, start, + length)); + } + } + return result; } /** - * Writes out the text representation of an integer using base 10 to an + * Writes out the text representation of an integer using base 10 to an * OutputStream in UTF-8 encoding. - * - * Note: division by a constant (like 10) is much faster than division by - * a variable. That's one of the reasons that we don't make radix a - * parameter here. - * - * @param out the outputstream to write to - * @param i an int to write out - * @throws IOException + * + * Note: division by a constant (like 10) is much faster than division by a + * variable. That's one of the reasons that we don't make radix a parameter + * here. + * + * @param out + * the outputstream to write to + * @param i + * an int to write out + * @throws IOException */ public static void writeUTF8(OutputStream out, int i) throws IOException { if (i == 0) { @@ -187,23 +194,23 @@ // of overflow here. i = -i; } - + int start = 1000000000; - while (i/start == 0) { + while (i / start == 0) { start /= 10; } - + while (start > 0) { out.write('0' - (i / start % 10)); start /= 10; } } - + public static void writeUTF8NoException(OutputStream out, int i) { try { writeUTF8(out, i); } catch (IOException e) { throw new RuntimeException(e); } - } + } } Index: serde/src/java/org/apache/hadoop/hive/serde2/binarysortable/BinarySortableSerDe.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/binarysortable/BinarySortableSerDe.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/binarysortable/BinarySortableSerDe.java (working copy) @@ -65,43 +65,41 @@ import org.apache.hadoop.io.Writable; /** - * BinarySortableSerDe can be used to write data in a way that the data can be - * compared byte-by-byte with the same order. + * BinarySortableSerDe can be used to write data in a way that the data can be + * compared byte-by-byte with the same order. * - * The data format: - * NULL: a single byte \0 - * NON-NULL Primitives: ALWAYS prepend a single byte \1, and then: - * Boolean: FALSE = \1, TRUE = \2 - * Byte: flip the sign-bit to make sure negative comes before positive - * Short: flip the sign-bit to make sure negative comes before positive - * Int: flip the sign-bit to make sure negative comes before positive - * Long: flip the sign-bit to make sure negative comes before positive - * Double: flip the sign-bit for positive double, and all bits for negative double values - * String: NULL-terminated UTF-8 string, with NULL escaped to \1 \1, and \1 escaped to \1 \2 - * NON-NULL Complex Types: ALWAYS prepend a single byte \1, and then: - * Struct: one field by one field. - * List: \1 followed by each element, and \0 to terminate - * Map: \1 followed by each key and then each value, and \0 to terminate - * - * This SerDe takes an additional parameter SERIALIZATION_SORT_ORDER which is a string containing only "+" and "-". - * The length of the string should equal to the number of fields in the top-level struct for serialization. - * "+" means the field should be sorted ascendingly, and "-" means descendingly. The sub fields in the same top-level - * field will have the same sort order. + * The data format: NULL: a single byte \0 NON-NULL Primitives: ALWAYS prepend a + * single byte \1, and then: Boolean: FALSE = \1, TRUE = \2 Byte: flip the + * sign-bit to make sure negative comes before positive Short: flip the sign-bit + * to make sure negative comes before positive Int: flip the sign-bit to make + * sure negative comes before positive Long: flip the sign-bit to make sure + * negative comes before positive Double: flip the sign-bit for positive double, + * and all bits for negative double values String: NULL-terminated UTF-8 string, + * with NULL escaped to \1 \1, and \1 escaped to \1 \2 NON-NULL Complex Types: + * ALWAYS prepend a single byte \1, and then: Struct: one field by one field. + * List: \1 followed by each element, and \0 to terminate Map: \1 followed by + * each key and then each value, and \0 to terminate * + * This SerDe takes an additional parameter SERIALIZATION_SORT_ORDER which is a + * string containing only "+" and "-". The length of the string should equal to + * the number of fields in the top-level struct for serialization. "+" means the + * field should be sorted ascendingly, and "-" means descendingly. The sub + * fields in the same top-level field will have the same sort order. + * */ public class BinarySortableSerDe implements SerDe { - public static final Log LOG = LogFactory.getLog( - BinarySortableSerDe.class.getName()); - + public static final Log LOG = LogFactory.getLog(BinarySortableSerDe.class + .getName()); + List columnNames; List columnTypes; - + TypeInfo rowTypeInfo; StructObjectInspector rowObjectInspector; - + boolean[] columnSortOrderIsDesc; - + @Override public void initialize(Configuration conf, Properties tbl) throws SerDeException { @@ -117,27 +115,30 @@ if (columnTypeProperty.length() == 0) { columnTypes = new ArrayList(); } else { - columnTypes = TypeInfoUtils.getTypeInfosFromTypeString(columnTypeProperty); + columnTypes = TypeInfoUtils + .getTypeInfosFromTypeString(columnTypeProperty); } - assert(columnNames.size() == columnTypes.size()); - + assert (columnNames.size() == columnTypes.size()); + // Create row related objects rowTypeInfo = TypeInfoFactory.getStructTypeInfo(columnNames, columnTypes); - rowObjectInspector = (StructObjectInspector)TypeInfoUtils + rowObjectInspector = (StructObjectInspector) TypeInfoUtils .getStandardWritableObjectInspectorFromTypeInfo(rowTypeInfo); row = new ArrayList(columnNames.size()); - for (int i=0; i getSerializedClass() { return BytesWritable.class; @@ -150,260 +151,268 @@ ArrayList row; InputByteBuffer inputByteBuffer = new InputByteBuffer(); + @Override public Object deserialize(Writable blob) throws SerDeException { - BytesWritable data = (BytesWritable)blob; + BytesWritable data = (BytesWritable) blob; inputByteBuffer.reset(data.get(), 0, data.getSize()); - + try { - for (int i=0; i r = reuse == null ? new ArrayList() : (ArrayList)reuse; - - // Read the list - int size = 0; - while (true) { - int more = buffer.read(invert); - if (more == 0) { - // \0 to terminate - break; - } - // \1 followed by each element - assert(more == 1); - if (size == r.size()) { - r.add(null); - } - r.set(size, deserialize(buffer, etype, invert, r.get(size))); - size++; + case LONG: { + LongWritable r = reuse == null ? new LongWritable() + : (LongWritable) reuse; + long v = buffer.read(invert) ^ 0x80; + for (int i = 0; i < 7; i++) { + v = (v << 8) + (buffer.read(invert) & 0xff); } - // Remove additional elements if the list is reused - while (r.size() > size) { - r.remove(r.size()-1); + r.set(v); + return r; + } + case FLOAT: { + FloatWritable r = reuse == null ? new FloatWritable() + : (FloatWritable) reuse; + int v = 0; + for (int i = 0; i < 4; i++) { + v = (v << 8) + (buffer.read(invert) & 0xff); } + if ((v & (1 << 31)) == 0) { + // negative number, flip all bits + v = ~v; + } else { + // positive number, flip the first bit + v = v ^ (1 << 31); + } + r.set(Float.intBitsToFloat(v)); return r; - } - case MAP: { - MapTypeInfo mtype = (MapTypeInfo)type; - TypeInfo ktype = mtype.getMapKeyTypeInfo(); - TypeInfo vtype = mtype.getMapValueTypeInfo(); - - // Create the map if needed - Map r; - if (reuse == null) { - r = new HashMap(); + } + case DOUBLE: { + DoubleWritable r = reuse == null ? new DoubleWritable() + : (DoubleWritable) reuse; + long v = 0; + for (int i = 0; i < 8; i++) { + v = (v << 8) + (buffer.read(invert) & 0xff); + } + if ((v & (1L << 63)) == 0) { + // negative number, flip all bits + v = ~v; } else { - r = (HashMap)reuse; - r.clear(); + // positive number, flip the first bit + v = v ^ (1L << 63); } - - // Read the map - int size = 0; - while (true) { - int more = buffer.read(invert); - if (more == 0) { - // \0 to terminate + r.set(Double.longBitsToDouble(v)); + return r; + } + case STRING: { + Text r = reuse == null ? new Text() : (Text) reuse; + // Get the actual length first + int start = buffer.tell(); + int length = 0; + do { + byte b = buffer.read(invert); + if (b == 0) { + // end of string break; } - // \1 followed by each key and then each value - assert(more == 1); - Object k = deserialize(buffer, ktype, invert, null); - Object v = deserialize(buffer, vtype, invert, null); - r.put(k, v); + if (b == 1) { + // the last char is an escape char. read the actual char + buffer.read(invert); + } + length++; + } while (true); + + if (length == buffer.tell() - start) { + // No escaping happened, so we are already done. + r.set(buffer.getData(), start, length); + } else { + // Escaping happened, we need to copy byte-by-byte. + // 1. Set the length first. + r.set(buffer.getData(), start, length); + // 2. Reset the pointer. + buffer.seek(start); + // 3. Copy the data. + byte[] rdata = r.getBytes(); + for (int i = 0; i < length; i++) { + byte b = buffer.read(invert); + if (b == 1) { + // The last char is an escape char, read the actual char. + // The serialization format escape \0 to \1, and \1 to \2, + // to make sure the string is null-terminated. + b = (byte) (buffer.read(invert) - 1); + } + rdata[i] = b; + } + // 4. Read the null terminator. + byte b = buffer.read(invert); + assert (b == 0); } return r; } - case STRUCT: { - StructTypeInfo stype = (StructTypeInfo)type; - List fieldTypes = stype.getAllStructFieldTypeInfos(); - int size = fieldTypes.size(); - // Create the struct if needed - ArrayList r = reuse == null ? new ArrayList(size) : (ArrayList)reuse; - assert(r.size() <= size); - // Set the size of the struct - while (r.size() < size) { + default: { + throw new RuntimeException("Unrecognized type: " + + ptype.getPrimitiveCategory()); + } + } + } + case LIST: { + ListTypeInfo ltype = (ListTypeInfo) type; + TypeInfo etype = ltype.getListElementTypeInfo(); + + // Create the list if needed + ArrayList r = reuse == null ? new ArrayList() + : (ArrayList) reuse; + + // Read the list + int size = 0; + while (true) { + int more = buffer.read(invert); + if (more == 0) { + // \0 to terminate + break; + } + // \1 followed by each element + assert (more == 1); + if (size == r.size()) { r.add(null); } - // Read one field by one field - for (int eid = 0; eid < size; eid++) { - r.set(eid, deserialize(buffer, fieldTypes.get(eid), invert, r.get(eid))); + r.set(size, deserialize(buffer, etype, invert, r.get(size))); + size++; + } + // Remove additional elements if the list is reused + while (r.size() > size) { + r.remove(r.size() - 1); + } + return r; + } + case MAP: { + MapTypeInfo mtype = (MapTypeInfo) type; + TypeInfo ktype = mtype.getMapKeyTypeInfo(); + TypeInfo vtype = mtype.getMapValueTypeInfo(); + + // Create the map if needed + Map r; + if (reuse == null) { + r = new HashMap(); + } else { + r = (HashMap) reuse; + r.clear(); + } + + while (true) { + int more = buffer.read(invert); + if (more == 0) { + // \0 to terminate + break; } - return r; + // \1 followed by each key and then each value + assert (more == 1); + Object k = deserialize(buffer, ktype, invert, null); + Object v = deserialize(buffer, vtype, invert, null); + r.put(k, v); } - default: { - throw new RuntimeException("Unrecognized type: " + type.getCategory()); + return r; + } + case STRUCT: { + StructTypeInfo stype = (StructTypeInfo) type; + List fieldTypes = stype.getAllStructFieldTypeInfos(); + int size = fieldTypes.size(); + // Create the struct if needed + ArrayList r = reuse == null ? new ArrayList(size) + : (ArrayList) reuse; + assert (r.size() <= size); + // Set the size of the struct + while (r.size() < size) { + r.add(null); } + // Read one field by one field + for (int eid = 0; eid < size; eid++) { + r + .set(eid, deserialize(buffer, fieldTypes.get(eid), invert, r + .get(eid))); + } + return r; } + default: { + throw new RuntimeException("Unrecognized type: " + type.getCategory()); + } + } } - + BytesWritable serializeBytesWritable = new BytesWritable(); OutputByteBuffer outputByteBuffer = new OutputByteBuffer(); - + @Override public Writable serialize(Object obj, ObjectInspector objInspector) throws SerDeException { outputByteBuffer.reset(); - StructObjectInspector soi = (StructObjectInspector)objInspector; + StructObjectInspector soi = (StructObjectInspector) objInspector; List fields = soi.getAllStructFieldRefs(); - - for (int i=0; i> 8) ^ 0x80), invert); - buffer.write((byte)v, invert); - return; - } - case INT: { - IntObjectInspector ioi = (IntObjectInspector)poi; - int v = ioi.get(o); - buffer.write((byte)((v >> 24) ^ 0x80), invert); - buffer.write((byte)(v >> 16), invert); - buffer.write((byte)(v >> 8), invert); - buffer.write((byte)v, invert); - return; - } - case LONG: { - LongObjectInspector loi = (LongObjectInspector)poi; - long v = loi.get(o); - buffer.write((byte)((v >> 56) ^ 0x80), invert); - buffer.write((byte)(v >> 48), invert); - buffer.write((byte)(v >> 40), invert); - buffer.write((byte)(v >> 32), invert); - buffer.write((byte)(v >> 24), invert); - buffer.write((byte)(v >> 16), invert); - buffer.write((byte)(v >> 8), invert); - buffer.write((byte)v, invert); - return; - } - case FLOAT: { - FloatObjectInspector foi = (FloatObjectInspector)poi; - int v = Float.floatToIntBits(foi.get(o)); - if ((v & (1<<31)) != 0) { - // negative number, flip all bits - v = ~v; - } else { - // positive number, flip the first bit - v = v ^ (1<<31); - } - buffer.write((byte)(v >> 24), invert); - buffer.write((byte)(v >> 16), invert); - buffer.write((byte)(v >> 8), invert); - buffer.write((byte)v, invert); - return; - } - case DOUBLE: { - DoubleObjectInspector doi = (DoubleObjectInspector)poi; - long v = Double.doubleToLongBits(doi.get(o)); - if ((v & (1L<<63)) != 0) { - // negative number, flip all bits - v = ~v; - } else { - // positive number, flip the first bit - v = v ^ (1L<<63); - } - buffer.write((byte)(v >> 56), invert); - buffer.write((byte)(v >> 48), invert); - buffer.write((byte)(v >> 40), invert); - buffer.write((byte)(v >> 32), invert); - buffer.write((byte)(v >> 24), invert); - buffer.write((byte)(v >> 16), invert); - buffer.write((byte)(v >> 8), invert); - buffer.write((byte)v, invert); - return; - } - case STRING: { - StringObjectInspector soi = (StringObjectInspector)poi; - Text t = soi.getPrimitiveWritableObject(o); - byte[] data = t.getBytes(); - int length = t.getLength(); - for (int i=0; i> 8) ^ 0x80), invert); + buffer.write((byte) v, invert); + return; + } + case INT: { + IntObjectInspector ioi = (IntObjectInspector) poi; + int v = ioi.get(o); + buffer.write((byte) ((v >> 24) ^ 0x80), invert); + buffer.write((byte) (v >> 16), invert); + buffer.write((byte) (v >> 8), invert); + buffer.write((byte) v, invert); + return; + } + case LONG: { + LongObjectInspector loi = (LongObjectInspector) poi; + long v = loi.get(o); + buffer.write((byte) ((v >> 56) ^ 0x80), invert); + buffer.write((byte) (v >> 48), invert); + buffer.write((byte) (v >> 40), invert); + buffer.write((byte) (v >> 32), invert); + buffer.write((byte) (v >> 24), invert); + buffer.write((byte) (v >> 16), invert); + buffer.write((byte) (v >> 8), invert); + buffer.write((byte) v, invert); + return; + } + case FLOAT: { + FloatObjectInspector foi = (FloatObjectInspector) poi; + int v = Float.floatToIntBits(foi.get(o)); + if ((v & (1 << 31)) != 0) { + // negative number, flip all bits + v = ~v; + } else { + // positive number, flip the first bit + v = v ^ (1 << 31); } - // and \0 to terminate - buffer.write((byte)0, invert); + buffer.write((byte) (v >> 24), invert); + buffer.write((byte) (v >> 16), invert); + buffer.write((byte) (v >> 8), invert); + buffer.write((byte) v, invert); return; - } - case MAP: { - MapObjectInspector moi = (MapObjectInspector)oi; - ObjectInspector koi = moi.getMapKeyObjectInspector(); - ObjectInspector voi = moi.getMapValueObjectInspector(); - - // \1 followed by each key and then each value - Map map = moi.getMap(o); - for(Map.Entry entry: map.entrySet()) { - buffer.write((byte)1, invert); - serialize(buffer, entry.getKey(), koi, invert); - serialize(buffer, entry.getValue(), voi, invert); + } + case DOUBLE: { + DoubleObjectInspector doi = (DoubleObjectInspector) poi; + long v = Double.doubleToLongBits(doi.get(o)); + if ((v & (1L << 63)) != 0) { + // negative number, flip all bits + v = ~v; + } else { + // positive number, flip the first bit + v = v ^ (1L << 63); } - // and \0 to terminate - buffer.write((byte)0, invert); + buffer.write((byte) (v >> 56), invert); + buffer.write((byte) (v >> 48), invert); + buffer.write((byte) (v >> 40), invert); + buffer.write((byte) (v >> 32), invert); + buffer.write((byte) (v >> 24), invert); + buffer.write((byte) (v >> 16), invert); + buffer.write((byte) (v >> 8), invert); + buffer.write((byte) v, invert); return; } - case STRUCT: { - StructObjectInspector soi = (StructObjectInspector)oi; - List fields = soi.getAllStructFieldRefs(); - - for (int i=0; i map = moi.getMap(o); + for (Map.Entry entry : map.entrySet()) { + buffer.write((byte) 1, invert); + serialize(buffer, entry.getKey(), koi, invert); + serialize(buffer, entry.getValue(), voi, invert); + } + // and \0 to terminate + buffer.write((byte) 0, invert); + return; + } + case STRUCT: { + StructObjectInspector soi = (StructObjectInspector) oi; + List fields = soi.getAllStructFieldRefs(); + + for (int i = 0; i < fields.size(); i++) { + serialize(buffer, soi.getStructFieldData(o, fields.get(i)), fields.get( + i).getFieldObjectInspector(), invert); + } + return; + } + default: { + throw new RuntimeException("Unrecognized type: " + oi.getCategory()); + } + } + } } Index: serde/src/java/org/apache/hadoop/hive/serde2/binarysortable/InputByteBuffer.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/binarysortable/InputByteBuffer.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/binarysortable/InputByteBuffer.java (working copy) @@ -21,15 +21,15 @@ import java.io.IOException; /** - * This class is much more efficient than ByteArrayInputStream - * because none of the methods are synchronized. + * This class is much more efficient than ByteArrayInputStream because none of + * the methods are synchronized. */ public class InputByteBuffer { - + byte[] data; int start; int end; - + /** * Reset the byte buffer to the given byte range. */ @@ -38,66 +38,63 @@ this.start = start; this.end = end; } - + public final byte read() throws IOException { return read(false); } - + /** - * Read one byte from the byte buffer. - * Final method to help inlining. - * @param invert whether we want to invert all the bits. + * Read one byte from the byte buffer. Final method to help inlining. + * + * @param invert + * whether we want to invert all the bits. */ public final byte read(boolean invert) throws IOException { if (start >= end) { throw new EOFException(); } if (invert) { - return (byte)(0xff ^ data[start++]); + return (byte) (0xff ^ data[start++]); } else { return data[start++]; } } - + /** - * Return the current position. - * Final method to help inlining. + * Return the current position. Final method to help inlining. */ public final int tell() { return start; } - + /** - * Set the current position. - * Final method to help inlining. + * Set the current position. Final method to help inlining. */ public final void seek(int position) { start = position; } - + public final int getEnd() { return end; } - + /** * Returns the underlying byte array. */ public final byte[] getData() { return data; } - + /** * Return the bytes in hex format. */ public String dumpHex() { StringBuilder sb = new StringBuilder(); - for (int i=start; i notSkipIDs = ColumnProjectionUtils.getReadColumnIDs(job); - + java.util.ArrayList notSkipIDs = ColumnProjectionUtils + .getReadColumnIDs(job); + cachedLazyStruct = new ColumnarStruct(cachedObjectInspector, notSkipIDs); - + int size = serdeParams.getColumnTypes().size(); field = new BytesRefWritable[size]; for (int i = 0; i < size; i++) { @@ -204,17 +205,17 @@ && (declaredFields == null || declaredFields.get(i) .getFieldObjectInspector().getCategory().equals( Category.PRIMITIVE))) { - LazySimpleSerDe.serialize(serializeStream, - SerDeUtils.getJSONString(f, foi), + LazySimpleSerDe.serialize(serializeStream, SerDeUtils.getJSONString( + f, foi), PrimitiveObjectInspectorFactory.javaStringObjectInspector, serdeParams.getSeparators(), 1, serdeParams.getNullSequence(), - serdeParams.isEscaped(), serdeParams.getEscapeChar(), - serdeParams.getNeedsEscape()); + serdeParams.isEscaped(), serdeParams.getEscapeChar(), serdeParams + .getNeedsEscape()); } else { - LazySimpleSerDe.serialize(serializeStream, f, foi, - serdeParams.getSeparators(), 1, serdeParams.getNullSequence(), - serdeParams.isEscaped(), serdeParams.getEscapeChar(), - serdeParams.getNeedsEscape()); + LazySimpleSerDe.serialize(serializeStream, f, foi, serdeParams + .getSeparators(), 1, serdeParams.getNullSequence(), serdeParams + .isEscaped(), serdeParams.getEscapeChar(), serdeParams + .getNeedsEscape()); } field[i].set(serializeStream.getData(), count, serializeStream Index: serde/src/java/org/apache/hadoop/hive/serde2/columnar/ColumnarStruct.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/columnar/ColumnarStruct.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/columnar/ColumnarStruct.java (working copy) @@ -20,9 +20,10 @@ import java.io.IOException; import java.util.ArrayList; -import java.util.Collections; import java.util.List; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hive.serde2.lazy.ByteArrayRef; import org.apache.hadoop.hive.serde2.lazy.LazyFactory; import org.apache.hadoop.hive.serde2.lazy.LazyObject; @@ -31,8 +32,6 @@ import org.apache.hadoop.hive.serde2.objectinspector.StructField; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; import org.apache.hadoop.io.Text; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; /** * ColumnarStruct is different from LazyStruct in that ColumnarStruct's field @@ -47,11 +46,11 @@ * The fields of the struct. */ LazyObject[] fields; - + private static final Log LOG = LogFactory.getLog(ColumnarStruct.class); - - int[] prjColIDs = null; // list of projected column IDs - + + int[] prjColIDs = null; // list of projected column IDs + /** * Construct a ColumnarStruct object with the TypeInfo. It creates the first * level object at the first place @@ -62,7 +61,7 @@ public ColumnarStruct(ObjectInspector oi) { this(oi, null); } - + /** * Construct a ColumnarStruct object with the TypeInfo. It creates the first * level object at the first place @@ -70,44 +69,50 @@ * @param oi * the ObjectInspector representing the type of this LazyStruct. * @param notSkippedColumnIDs - * the column ids that should not be skipped + * the column ids that should not be skipped */ - public ColumnarStruct(ObjectInspector oi, ArrayList notSkippedColumnIDs) { - List fieldRefs = ((StructObjectInspector) oi).getAllStructFieldRefs(); + public ColumnarStruct(ObjectInspector oi, + ArrayList notSkippedColumnIDs) { + List fieldRefs = ((StructObjectInspector) oi) + .getAllStructFieldRefs(); int num = fieldRefs.size(); fields = new LazyObject[num]; cachedByteArrayRef = new ByteArrayRef[num]; rawBytesField = new BytesRefWritable[num]; fieldSkipped = new boolean[num]; inited = new boolean[num]; - - //if no columns is set to be skipped, add all columns in 'notSkippedColumnIDs' - if (notSkippedColumnIDs == null || notSkippedColumnIDs.size() == 0) { - for (int i = 0; i < num; i++) - notSkippedColumnIDs.add(i); - } - + + // if no columns is set to be skipped, add all columns in + // 'notSkippedColumnIDs' + if (notSkippedColumnIDs == null || notSkippedColumnIDs.size() == 0) { + for (int i = 0; i < num; i++) { + notSkippedColumnIDs.add(i); + } + } + for (int i = 0; i < num; i++) { - fields[i] = LazyFactory.createLazyObject(fieldRefs.get(i).getFieldObjectInspector()); + fields[i] = LazyFactory.createLazyObject(fieldRefs.get(i) + .getFieldObjectInspector()); cachedByteArrayRef[i] = new ByteArrayRef(); - if(!notSkippedColumnIDs.contains(i)){ - fieldSkipped[i] = true; - inited[i] = true; - } else - inited[i] = false; + if (!notSkippedColumnIDs.contains(i)) { + fieldSkipped[i] = true; + inited[i] = true; + } else { + inited[i] = false; + } } - - // maintain a list of non-NULL column IDs - int min = notSkippedColumnIDs.size() > num ? num : notSkippedColumnIDs - .size(); - prjColIDs = new int[min]; - for (int i = 0, index = 0; i < notSkippedColumnIDs.size(); ++i) { - int readCol = notSkippedColumnIDs.get(i).intValue(); - if (readCol < num) { - prjColIDs[index] = readCol; - index++; - } - } + + // maintain a list of non-NULL column IDs + int min = notSkippedColumnIDs.size() > num ? num : notSkippedColumnIDs + .size(); + prjColIDs = new int[min]; + for (int i = 0, index = 0; i < notSkippedColumnIDs.size(); ++i) { + int readCol = notSkippedColumnIDs.get(i).intValue(); + if (readCol < num) { + prjColIDs[index] = readCol; + index++; + } + } } /** @@ -152,8 +157,9 @@ * @return The value of the field */ protected Object uncheckedGetField(int fieldID, Text nullSequence) { - if (fieldSkipped[fieldID]) + if (fieldSkipped[fieldID]) { return null; + } if (!inited[fieldID]) { BytesRefWritable passedInField = rawBytesField[fieldID]; try { @@ -165,35 +171,36 @@ .getStart(), passedInField.getLength()); inited[fieldID] = true; } - + byte[] data = cachedByteArrayRef[fieldID].getData(); int fieldLen = rawBytesField[fieldID].length; - + if (fieldLen == nullSequence.getLength() - && LazyUtils.compare(data, rawBytesField[fieldID].getStart(), - fieldLen, nullSequence.getBytes(), 0, nullSequence.getLength()) == 0) { + && LazyUtils.compare(data, rawBytesField[fieldID].getStart(), fieldLen, + nullSequence.getBytes(), 0, nullSequence.getLength()) == 0) { return null; } return fields[fieldID].getObject(); } - - /* ============================ [PERF] =================================== - * This function is called for every row. Setting up the selected/projected - * columns at the first call, and don't do that for the following calls. - * Ideally this should be done in the constructor where we don't need to - * branch in the function for each row. - * ========================================================================= + + /* + * ============================ [PERF] =================================== + * This function is called for every row. Setting up the selected/projected + * columns at the first call, and don't do that for the following calls. + * Ideally this should be done in the constructor where we don't need to + * branch in the function for each row. + * ========================================================================= */ public void init(BytesRefArrayWritable cols) { - for (int i = 0; i < prjColIDs.length; ++i ) { + for (int i = 0; i < prjColIDs.length; ++i) { int fieldIndex = prjColIDs[i]; - if(fieldIndex < cols.size()){ - rawBytesField[fieldIndex] = cols.unCheckedGet(fieldIndex); - inited[fieldIndex] = false; + if (fieldIndex < cols.size()) { + rawBytesField[fieldIndex] = cols.unCheckedGet(fieldIndex); + inited[fieldIndex] = false; } else { - // select columns that actually do not exist in the file. - fieldSkipped[fieldIndex] = true; + // select columns that actually do not exist in the file. + fieldSkipped[fieldIndex] = true; } } } Index: serde/src/java/org/apache/hadoop/hive/serde2/columnar/LazyDecompressionCallback.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/columnar/LazyDecompressionCallback.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/columnar/LazyDecompressionCallback.java (working copy) @@ -20,9 +20,8 @@ import java.io.IOException; - /** - * Used to call back lazy decompression process. + * Used to call back lazy decompression process. * * @see #BytesRefWritable */ Index: serde/src/java/org/apache/hadoop/hive/serde2/columnar/BytesRefArrayWritable.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/columnar/BytesRefArrayWritable.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/columnar/BytesRefArrayWritable.java (working copy) @@ -54,8 +54,9 @@ * if the specified initial capacity is negative */ public BytesRefArrayWritable(int capacity) { - if (capacity < 0) + if (capacity < 0) { throw new IllegalArgumentException("Capacity can not be negative."); + } bytesRefWritables = new BytesRefWritable[0]; ensureCapacity(capacity); } @@ -85,10 +86,10 @@ * @throws IndexOutOfBoundsException */ public BytesRefWritable get(int index) { - if (index >= valid) + if (index >= valid) { throw new IndexOutOfBoundsException( - "This BytesRefArrayWritable only has " + valid - + " valid values."); + "This BytesRefArrayWritable only has " + valid + " valid values."); + } return bytesRefWritables[index]; } @@ -115,12 +116,14 @@ * if the specified new element is null */ public void set(int index, BytesRefWritable bytesRefWritable) { - if (bytesRefWritable == null) + if (bytesRefWritable == null) { throw new IllegalArgumentException("Can not assign null."); + } ensureCapacity(index + 1); bytesRefWritables[index] = bytesRefWritable; - if (valid <= index) + if (valid <= index) { valid = index + 1; + } } /** @@ -128,18 +131,22 @@ */ @Override public int compareTo(BytesRefArrayWritable other) { - if (other == null) + if (other == null) { throw new IllegalArgumentException("Argument can not be null."); - if (this == other) + } + if (this == other) { return 0; + } int sizeDiff = valid - other.valid; - if (sizeDiff != 0) + if (sizeDiff != 0) { return sizeDiff; + } for (int i = 0; i < valid; i++) { - if (other.contains(bytesRefWritables[i])) + if (other.contains(bytesRefWritables[i])) { continue; - else + } else { return 1; + } } return 0; } @@ -155,11 +162,13 @@ * if the specified element is null */ public boolean contains(BytesRefWritable bytesRefWritable) { - if (bytesRefWritable == null) + if (bytesRefWritable == null) { throw new IllegalArgumentException("Argument can not be null."); + } for (int i = 0; i < valid; i++) { - if (bytesRefWritables[i].equals(bytesRefWritable)) + if (bytesRefWritables[i].equals(bytesRefWritable)) { return true; + } } return false; } @@ -167,9 +176,11 @@ /** * {@inheritDoc} */ + @Override public boolean equals(Object o) { - if (o == null || !(o instanceof BytesRefArrayWritable)) + if (o == null || !(o instanceof BytesRefArrayWritable)) { return false; + } return compareTo((BytesRefArrayWritable) o) == 0; } Index: serde/src/java/org/apache/hadoop/hive/serde2/columnar/BytesRefWritable.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/columnar/BytesRefWritable.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/columnar/BytesRefWritable.java (working copy) @@ -39,9 +39,8 @@ int start = 0; int length = 0; byte[] bytes = null; - + LazyDecompressionCallback lazyDecompressObj; - /** * Create a zero-size bytes. @@ -78,7 +77,7 @@ start = offset; length = len; } - + /** * Create a BytesRefWritable referenced to one section of the given bytes. The * argument lazyDecompressData refers to a LazyDecompressionCallback @@ -86,7 +85,8 @@ * uncompressed bytes of lazyDecompressData. Use offset and * len after uncompressing the data. */ - public BytesRefWritable(LazyDecompressionCallback lazyDecompressData, int offset, int len) { + public BytesRefWritable(LazyDecompressionCallback lazyDecompressData, + int offset, int len) { lazyDecompressObj = lazyDecompressData; start = offset; length = len; @@ -102,7 +102,7 @@ * Returns a copy of the underlying bytes referenced by this instance. * * @return a new copied byte array - * @throws IOException + * @throws IOException */ public byte[] getBytesCopy() throws IOException { lazyDecompress(); @@ -113,7 +113,8 @@ /** * Returns the underlying bytes. - * @throws IOException + * + * @throws IOException */ public byte[] getData() throws IOException { lazyDecompress(); @@ -132,7 +133,7 @@ length = len; lazyDecompressObj = null; } - + /** * readFields() will corrupt the array. So use the set method whenever * possible. @@ -176,11 +177,13 @@ } /** {@inheritDoc} */ + @Override public int hashCode() { return super.hashCode(); } /** {@inheritDoc} */ + @Override public String toString() { StringBuffer sb = new StringBuffer(3 * length); for (int idx = start; idx < length; idx++) { @@ -201,22 +204,26 @@ /** {@inheritDoc} */ @Override public int compareTo(BytesRefWritable other) { - if (other == null) + if (other == null) { throw new IllegalArgumentException("Argument can not be null."); - if (this == other) + } + if (this == other) { return 0; + } try { - return WritableComparator.compareBytes(getData(), start, getLength(), other - .getData(), other.start, other.getLength()); + return WritableComparator.compareBytes(getData(), start, getLength(), + other.getData(), other.start, other.getLength()); } catch (IOException e) { throw new RuntimeException(e); } } /** {@inheritDoc} */ + @Override public boolean equals(Object right_obj) { - if (right_obj == null || !(right_obj instanceof BytesRefWritable)) + if (right_obj == null || !(right_obj instanceof BytesRefWritable)) { return false; + } return compareTo((BytesRefWritable) right_obj) == 0; } Index: serde/src/java/org/apache/hadoop/hive/serde2/SerDe.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/SerDe.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/SerDe.java (working copy) @@ -21,12 +21,12 @@ /** * A union of HiveDeserializer and HiveSerializer interface. * - * If a developer wants his hive table to be read-only, then he just want to - * return + * If a developer wants his hive table to be read-only, then he just want to + * return * * both readable and writable, then - * - * + * + * */ public interface SerDe extends Deserializer, Serializer { Index: serde/src/java/org/apache/hadoop/hive/serde2/MetadataTypedColumnsetSerDe.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/MetadataTypedColumnsetSerDe.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/MetadataTypedColumnsetSerDe.java (working copy) @@ -38,35 +38,38 @@ import org.apache.hadoop.io.Text; import org.apache.hadoop.io.Writable; - public class MetadataTypedColumnsetSerDe implements SerDe { - public static final Log LOG = LogFactory.getLog(MetadataTypedColumnsetSerDe.class.getName()); + public static final Log LOG = LogFactory + .getLog(MetadataTypedColumnsetSerDe.class.getName()); static { StackTraceElement[] sTrace = new Exception().getStackTrace(); String className = sTrace[0].getClassName(); try { - // For backward compatibility: this class replaces the columnsetSerDe class. - SerDeUtils.registerSerDe("org.apache.hadoop.hive.serde.thrift.columnsetSerDe", - Class.forName(className)); - } catch(Exception e) { + // For backward compatibility: this class replaces the columnsetSerDe + // class. + SerDeUtils.registerSerDe( + "org.apache.hadoop.hive.serde.thrift.columnsetSerDe", Class + .forName(className)); + } catch (Exception e) { throw new RuntimeException(e); } } - + final public static String DefaultSeparator = "\001"; private String separator; final public static String defaultNullString = "\\N"; - private String nullString; + private String nullString; private List columnNames; private ObjectInspector cachedObjectInspector; private boolean lastColumnTakesRest = false; private int splitLimit = -1; - + + @Override public String toString() { return "MetaDataTypedColumnsetSerDe[" + separator + "," + columnNames + "]"; } @@ -78,17 +81,18 @@ private String getByteValue(String altValue, String defaultVal) { if (altValue != null && altValue.length() > 0) { try { - byte b [] = new byte[1]; + byte b[] = new byte[1]; b[0] = Byte.valueOf(altValue).byteValue(); return new String(b); - } catch(NumberFormatException e) { + } catch (NumberFormatException e) { return altValue; } } return defaultVal; } - public void initialize(Configuration job, Properties tbl) throws SerDeException { + public void initialize(Configuration job, Properties tbl) + throws SerDeException { String alt_sep = tbl.getProperty(Constants.SERIALIZATION_FORMAT); separator = getByteValue(alt_sep, DefaultSeparator); @@ -97,34 +101,46 @@ String columnProperty = tbl.getProperty("columns"); String serdeName = tbl.getProperty(Constants.SERIALIZATION_LIB); - // tables that were serialized with columnsetSerDe doesn't have metadata - // so this hack applies to all such tables + // tables that were serialized with columnsetSerDe doesn't have metadata + // so this hack applies to all such tables boolean columnsetSerDe = false; - if ((serdeName != null) && serdeName.equals("org.apache.hadoop.hive.serde.thrift.columnsetSerDe")) { + if ((serdeName != null) + && serdeName + .equals("org.apache.hadoop.hive.serde.thrift.columnsetSerDe")) { columnsetSerDe = true; } - if (columnProperty == null || columnProperty.length() == 0 + if (columnProperty == null || columnProperty.length() == 0 || columnsetSerDe) { // Hack for tables with no columns // Treat it as a table with a single column called "col" - cachedObjectInspector = ObjectInspectorFactory.getReflectionObjectInspector( - ColumnSet.class, ObjectInspectorFactory.ObjectInspectorOptions.JAVA); + cachedObjectInspector = ObjectInspectorFactory + .getReflectionObjectInspector(ColumnSet.class, + ObjectInspectorFactory.ObjectInspectorOptions.JAVA); } else { columnNames = Arrays.asList(columnProperty.split(",")); - cachedObjectInspector = MetadataListStructObjectInspector.getInstance(columnNames); + cachedObjectInspector = MetadataListStructObjectInspector + .getInstance(columnNames); } - - String lastColumnTakesRestString = tbl.getProperty(Constants.SERIALIZATION_LAST_COLUMN_TAKES_REST); - lastColumnTakesRest = (lastColumnTakesRestString != null && lastColumnTakesRestString.equalsIgnoreCase("true")); - splitLimit = (lastColumnTakesRest && columnNames != null) ? columnNames.size() : -1; - - LOG.debug(getClass().getName() + ": initialized with columnNames: " + columnNames + " and separator code=" + (int)separator.charAt(0) - + " lastColumnTakesRest=" + lastColumnTakesRest + " splitLimit=" + splitLimit); + + String lastColumnTakesRestString = tbl + .getProperty(Constants.SERIALIZATION_LAST_COLUMN_TAKES_REST); + lastColumnTakesRest = (lastColumnTakesRestString != null && lastColumnTakesRestString + .equalsIgnoreCase("true")); + splitLimit = (lastColumnTakesRest && columnNames != null) ? columnNames + .size() : -1; + + LOG.debug(getClass().getName() + ": initialized with columnNames: " + + columnNames + " and separator code=" + (int) separator.charAt(0) + + " lastColumnTakesRest=" + lastColumnTakesRest + " splitLimit=" + + splitLimit); } - + /** * Split the row into columns. - * @param limit up to limit columns will be produced (the last column takes all the rest), -1 for unlimited. + * + * @param limit + * up to limit columns will be produced (the last column takes all + * the rest), -1 for unlimited. * @return The ColumnSet object * @throws Exception */ @@ -135,9 +151,9 @@ } else { c.col.clear(); } - String [] l1 = row.split(sep, limit); + String[] l1 = row.split(sep, limit); - for(String s: l1) { + for (String s : l1) { if (s.equals(nullString)) { c.col.add(null); } else { @@ -146,12 +162,13 @@ } return (c); } - + ColumnSet deserializeCache = new ColumnSet(); + public Object deserialize(Writable field) throws SerDeException { String row = null; if (field instanceof BytesWritable) { - BytesWritable b = (BytesWritable)field; + BytesWritable b = (BytesWritable) field; try { row = Text.decode(b.get(), 0, b.getSize()); } catch (CharacterCodingException e) { @@ -163,17 +180,17 @@ try { deserialize(deserializeCache, row, separator, nullString, splitLimit); if (columnNames != null) { - assert(columnNames.size() == deserializeCache.col.size()); + assert (columnNames.size() == deserializeCache.col.size()); } return deserializeCache; } catch (ClassCastException e) { - throw new SerDeException( this.getClass().getName() + " expects Text or BytesWritable", e); + throw new SerDeException(this.getClass().getName() + + " expects Text or BytesWritable", e); } catch (Exception e) { throw new SerDeException(e); } } - - + public ObjectInspector getObjectInspector() throws SerDeException { return cachedObjectInspector; } @@ -181,27 +198,33 @@ public Class getSerializedClass() { return Text.class; } - + Text serializeCache = new Text(); - public Writable serialize(Object obj, ObjectInspector objInspector) throws SerDeException { + public Writable serialize(Object obj, ObjectInspector objInspector) + throws SerDeException { + if (objInspector.getCategory() != Category.STRUCT) { - throw new SerDeException(getClass().toString() - + " can only serialize struct types, but we got: " + objInspector.getTypeName()); + throw new SerDeException(getClass().toString() + + " can only serialize struct types, but we got: " + + objInspector.getTypeName()); } StructObjectInspector soi = (StructObjectInspector) objInspector; List fields = soi.getAllStructFieldRefs(); - + StringBuilder sb = new StringBuilder(); - for(int i=0; i0) sb.append(separator); + for (int i = 0; i < fields.size(); i++) { + if (i > 0) { + sb.append(separator); + } Object column = soi.getStructFieldData(obj, fields.get(i)); if (fields.get(i).getFieldObjectInspector().getCategory() == Category.PRIMITIVE) { // For primitive object, serialize to plain string sb.append(column == null ? nullString : column.toString()); } else { // For complex object, serialize to JSON format - sb.append(SerDeUtils.getJSONString(column, fields.get(i).getFieldObjectInspector())); + sb.append(SerDeUtils.getJSONString(column, fields.get(i) + .getFieldObjectInspector())); } } serializeCache.set(sb.toString()); Index: serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryInteger.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryInteger.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryInteger.java (working copy) @@ -24,9 +24,11 @@ /** * LazyBinaryObject for integer which is serialized as VInt - * @see LazyBinaryUtils#readVInt(byte[], int, VInt) + * + * @see LazyBinaryUtils#readVInt(byte[], int, VInt) */ -public class LazyBinaryInteger extends LazyBinaryPrimitive { +public class LazyBinaryInteger extends + LazyBinaryPrimitive { LazyBinaryInteger(WritableIntObjectInspector oi) { super(oi); @@ -39,14 +41,14 @@ } /** - * The reusable vInt for decoding the integer + * The reusable vInt for decoding the integer */ VInt vInt = new LazyBinaryUtils.VInt(); - + @Override public void init(ByteArrayRef bytes, int start, int length) { LazyBinaryUtils.readVInt(bytes.getData(), start, vInt); - assert(length == vInt.length); + assert (length == vInt.length); data.set(vInt.value); } } Index: serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryLong.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryLong.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryLong.java (working copy) @@ -24,29 +24,31 @@ /** * LazyBinaryObject for long which stores as VLong. + * * @see LazyBinaryUtils#readVLong(byte[], int, VLong) */ -public class LazyBinaryLong extends LazyBinaryPrimitive { +public class LazyBinaryLong extends + LazyBinaryPrimitive { LazyBinaryLong(WritableLongObjectInspector oi) { super(oi); data = new LongWritable(); } - + LazyBinaryLong(LazyBinaryLong copy) { super(copy); data = new LongWritable(copy.data.get()); } /** - * The reusable vLong for decoding the long + * The reusable vLong for decoding the long */ VLong vLong = new LazyBinaryUtils.VLong(); - + @Override public void init(ByteArrayRef bytes, int start, int length) { LazyBinaryUtils.readVLong(bytes.getData(), start, vLong); - assert(length == vLong.length); + assert (length == vLong.length); data.set(vLong.value); } } Index: serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryShort.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryShort.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryShort.java (working copy) @@ -24,7 +24,8 @@ /** * LazyBinaryObject for short which takes two bytes. */ -public class LazyBinaryShort extends LazyBinaryPrimitive { +public class LazyBinaryShort extends + LazyBinaryPrimitive { LazyBinaryShort(WritableShortObjectInspector oi) { super(oi); @@ -38,7 +39,7 @@ @Override public void init(ByteArrayRef bytes, int start, int length) { - assert(2 == length); - data.set(LazyBinaryUtils.byteArrayToShort(bytes.getData(), start)); - } + assert (2 == length); + data.set(LazyBinaryUtils.byteArrayToShort(bytes.getData(), start)); + } } Index: serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryUtils.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryUtils.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryUtils.java (working copy) @@ -23,8 +23,8 @@ import org.apache.hadoop.hive.serde2.ByteStream.Output; import org.apache.hadoop.hive.serde2.lazybinary.objectinspector.LazyBinaryObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; @@ -36,206 +36,230 @@ import org.apache.hadoop.io.WritableUtils; public class LazyBinaryUtils { - + /** - * Convert the byte array to an int starting from the given offset. - * Refer to code by aeden on DZone Snippets: - * @param b the byte array - * @param offset the array offset - * @return the integer + * Convert the byte array to an int starting from the given offset. Refer to + * code by aeden on DZone Snippets: + * + * @param b + * the byte array + * @param offset + * the array offset + * @return the integer */ public static int byteArrayToInt(byte[] b, int offset) { - int value = 0; - for (int i = 0; i < 4; i++) { - int shift = (4 - 1 - i) * 8; - value += (b[i + offset] & 0x000000FF) << shift; - } - return value; + int value = 0; + for (int i = 0; i < 4; i++) { + int shift = (4 - 1 - i) * 8; + value += (b[i + offset] & 0x000000FF) << shift; + } + return value; } - + /** * Convert the byte array to a long starting from the given offset. - * @param b the byte array - * @param offset the array offset - * @return the long + * + * @param b + * the byte array + * @param offset + * the array offset + * @return the long */ public static long byteArrayToLong(byte[] b, int offset) { - long value = 0; - for (int i = 0; i < 8; i++) { - int shift = (8 - 1 - i) * 8; - value += ((long) (b[i + offset] & 0x00000000000000FF)) << shift; - } - return value; + long value = 0; + for (int i = 0; i < 8; i++) { + int shift = (8 - 1 - i) * 8; + value += ((long) (b[i + offset] & 0x00000000000000FF)) << shift; + } + return value; } /** * Convert the byte array to a short starting from the given offset. - * @param b the byte array - * @param offset the array offset - * @return the short + * + * @param b + * the byte array + * @param offset + * the array offset + * @return the short */ public static short byteArrayToShort(byte[] b, int offset) { - short value = 0; - value += (b[offset ] & 0x000000FF) << 8; - value += (b[offset+1] & 0x000000FF); - return value; + short value = 0; + value += (b[offset] & 0x000000FF) << 8; + value += (b[offset + 1] & 0x000000FF); + return value; } - + /** - * Record is the unit that data is serialized in. - * A record includes two parts. The first part stores the - * size of the element and the second part stores the - * real element. - * size element - * record -> |----|-------------------------| - * - * A RecordInfo stores two information of a record, - * the size of the "size" part which is the element offset - * and the size of the element part which is element size. - */ + * Record is the unit that data is serialized in. A record includes two parts. + * The first part stores the size of the element and the second part stores + * the real element. size element record -> |----|-------------------------| + * + * A RecordInfo stores two information of a record, the size of the "size" + * part which is the element offset and the size of the element part which is + * element size. + */ public static class RecordInfo { - public RecordInfo () { + public RecordInfo() { elementOffset = 0; - elementSize = 0; + elementSize = 0; } + public byte elementOffset; - public int elementSize; + public int elementSize; } static VInt vInt = new LazyBinaryUtils.VInt(); + /** - * Check a particular field and set its size and offset in bytes - * based on the field type and the bytes arrays. - * - * For void, boolean, byte, short, int, long, float and double, - * there is no offset and the size is fixed. For string, map, - * list, struct, the first four bytes are used to store the size. - * So the offset is 4 and the size is computed by concating the - * first four bytes together. The first four bytes are defined - * with respect to the offset in the bytes arrays. + * Check a particular field and set its size and offset in bytes based on the + * field type and the bytes arrays. * - * @param objectInspector object inspector of the field - * @param bytes bytes arrays store the table row - * @param offset offset of this field - * @param recordInfo modify this byteinfo object and return it + * For void, boolean, byte, short, int, long, float and double, there is no + * offset and the size is fixed. For string, map, list, struct, the first four + * bytes are used to store the size. So the offset is 4 and the size is + * computed by concating the first four bytes together. The first four bytes + * are defined with respect to the offset in the bytes arrays. + * + * @param objectInspector + * object inspector of the field + * @param bytes + * bytes arrays store the table row + * @param offset + * offset of this field + * @param recordInfo + * modify this byteinfo object and return it */ - public static void checkObjectByteInfo(ObjectInspector objectInspector, byte[] bytes, int offset, RecordInfo recordInfo) { - Category category = objectInspector.getCategory(); + public static void checkObjectByteInfo(ObjectInspector objectInspector, + byte[] bytes, int offset, RecordInfo recordInfo) { + Category category = objectInspector.getCategory(); switch (category) { case PRIMITIVE: - PrimitiveCategory primitiveCategory = ((PrimitiveObjectInspector)objectInspector).getPrimitiveCategory(); + PrimitiveCategory primitiveCategory = ((PrimitiveObjectInspector) objectInspector) + .getPrimitiveCategory(); switch (primitiveCategory) { case VOID: case BOOLEAN: case BYTE: recordInfo.elementOffset = 0; - recordInfo.elementSize = 1; + recordInfo.elementSize = 1; break; case SHORT: recordInfo.elementOffset = 0; - recordInfo.elementSize = 2; + recordInfo.elementSize = 2; break; case FLOAT: recordInfo.elementOffset = 0; - recordInfo.elementSize = 4; - break; + recordInfo.elementSize = 4; + break; case DOUBLE: recordInfo.elementOffset = 0; - recordInfo.elementSize = 8; - break; + recordInfo.elementSize = 8; + break; case INT: recordInfo.elementOffset = 0; - recordInfo.elementSize = WritableUtils.decodeVIntSize(bytes[offset]); + recordInfo.elementSize = WritableUtils.decodeVIntSize(bytes[offset]); break; case LONG: recordInfo.elementOffset = 0; - recordInfo.elementSize = WritableUtils.decodeVIntSize(bytes[offset]); + recordInfo.elementSize = WritableUtils.decodeVIntSize(bytes[offset]); break; case STRING: // using vint instead of 4 bytes LazyBinaryUtils.readVInt(bytes, offset, vInt); recordInfo.elementOffset = vInt.length; - recordInfo.elementSize = vInt.value; + recordInfo.elementSize = vInt.value; break; - default: { - throw new RuntimeException("Unrecognized primitive type: " + primitiveCategory); - } + default: { + throw new RuntimeException("Unrecognized primitive type: " + + primitiveCategory); } + } break; case LIST: case MAP: case STRUCT: recordInfo.elementOffset = 4; - recordInfo.elementSize = LazyBinaryUtils.byteArrayToInt(bytes, offset); + recordInfo.elementSize = LazyBinaryUtils.byteArrayToInt(bytes, offset); break; - default : { - throw new RuntimeException("Unrecognized non-primitive type: " + category); - } + default: { + throw new RuntimeException("Unrecognized non-primitive type: " + category); } + } } - + /** - * A zero-compressed encoded long + * A zero-compressed encoded long */ public static class VLong { public VLong() { - value = 0; + value = 0; length = 0; } + public long value; - public byte length; + public byte length; }; - + /** * Reads a zero-compressed encoded long from a byte array and returns it. - * @param bytes the byte array - * @param offset offset of the array to read from - * @param vlong storing the deserialized long and its size in byte + * + * @param bytes + * the byte array + * @param offset + * offset of the array to read from + * @param vlong + * storing the deserialized long and its size in byte */ public static void readVLong(byte[] bytes, int offset, VLong vlong) { byte firstByte = bytes[offset]; - vlong.length = (byte)WritableUtils.decodeVIntSize(firstByte); + vlong.length = (byte) WritableUtils.decodeVIntSize(firstByte); if (vlong.length == 1) { vlong.value = firstByte; return; } long i = 0; - for (int idx = 0; idx < vlong.length-1; idx++) { - byte b = bytes[offset+1+idx]; + for (int idx = 0; idx < vlong.length - 1; idx++) { + byte b = bytes[offset + 1 + idx]; i = i << 8; i = i | (b & 0xFF); } vlong.value = (WritableUtils.isNegativeVInt(firstByte) ? (i ^ -1L) : i); } - + /** - * A zero-compressed encoded integer + * A zero-compressed encoded integer */ public static class VInt { public VInt() { - value = 0; + value = 0; length = 0; } - public int value; - public byte length; + + public int value; + public byte length; }; /** * Reads a zero-compressed encoded int from a byte array and returns it. - * @param bytes the byte array - * @param offset offset of the array to read from - * @param vInt storing the deserialized int and its size in byte + * + * @param bytes + * the byte array + * @param offset + * offset of the array to read from + * @param vInt + * storing the deserialized int and its size in byte */ public static void readVInt(byte[] bytes, int offset, VInt vInt) { byte firstByte = bytes[offset]; - vInt.length = (byte)WritableUtils.decodeVIntSize(firstByte); + vInt.length = (byte) WritableUtils.decodeVIntSize(firstByte); if (vInt.length == 1) { vInt.value = firstByte; return; } int i = 0; - for (int idx = 0; idx < vInt.length-1; idx++) { - byte b = bytes[offset+1+idx]; + for (int idx = 0; idx < vInt.length - 1; idx++) { + byte b = bytes[offset + 1 + idx]; i = i << 8; i = i | (b & 0xFF); } @@ -244,92 +268,111 @@ /** * Writes a zero-compressed encoded int to a byte array. - * @param byteStream the byte array/stream - * @param i the int + * + * @param byteStream + * the byte array/stream + * @param i + * the int */ public static void writeVInt(Output byteStream, int i) { writeVLong(byteStream, i); } - + /** * Write a zero-compressed encoded long to a byte array. - * @param byteStream the byte array/stream - * @param l the long + * + * @param byteStream + * the byte array/stream + * @param l + * the long */ public static void writeVLong(Output byteStream, long l) { if (l >= -112 && l <= 127) { - byteStream.write((byte)l); + byteStream.write((byte) l); return; } - + int len = -112; if (l < 0) { l ^= -1L; // take one's complement' len = -120; } - + long tmp = l; while (tmp != 0) { tmp = tmp >> 8; len--; } - - byteStream.write((byte)len); - + + byteStream.write((byte) len); + len = (len < -120) ? -(len + 120) : -(len + 112); - + for (int idx = len; idx != 0; idx--) { int shiftbits = (idx - 1) * 8; long mask = 0xFFL << shiftbits; - byteStream.write((byte)((l & mask) >> shiftbits)); + byteStream.write((byte) ((l & mask) >> shiftbits)); } - } - + } + static HashMap cachedLazyBinaryObjectInspector = new HashMap(); /** - * Returns the lazy binary object inspector that can be used to inspect an + * Returns the lazy binary object inspector that can be used to inspect an * lazy binary object of that typeInfo * - * For primitive types, we use the standard writable object inspector. + * For primitive types, we use the standard writable object inspector. */ - public static ObjectInspector getLazyBinaryObjectInspectorFromTypeInfo(TypeInfo typeInfo) { + public static ObjectInspector getLazyBinaryObjectInspectorFromTypeInfo( + TypeInfo typeInfo) { ObjectInspector result = cachedLazyBinaryObjectInspector.get(typeInfo); if (result == null) { - switch(typeInfo.getCategory()) { - case PRIMITIVE: { - result = PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector( - ((PrimitiveTypeInfo)typeInfo).getPrimitiveCategory()); - break; + switch (typeInfo.getCategory()) { + case PRIMITIVE: { + result = PrimitiveObjectInspectorFactory + .getPrimitiveWritableObjectInspector(((PrimitiveTypeInfo) typeInfo) + .getPrimitiveCategory()); + break; + } + case LIST: { + ObjectInspector elementObjectInspector = getLazyBinaryObjectInspectorFromTypeInfo(((ListTypeInfo) typeInfo) + .getListElementTypeInfo()); + result = LazyBinaryObjectInspectorFactory + .getLazyBinaryListObjectInspector(elementObjectInspector); + break; + } + case MAP: { + MapTypeInfo mapTypeInfo = (MapTypeInfo) typeInfo; + ObjectInspector keyObjectInspector = getLazyBinaryObjectInspectorFromTypeInfo(mapTypeInfo + .getMapKeyTypeInfo()); + ObjectInspector valueObjectInspector = getLazyBinaryObjectInspectorFromTypeInfo(mapTypeInfo + .getMapValueTypeInfo()); + result = LazyBinaryObjectInspectorFactory + .getLazyBinaryMapObjectInspector(keyObjectInspector, + valueObjectInspector); + break; + } + case STRUCT: { + StructTypeInfo structTypeInfo = (StructTypeInfo) typeInfo; + List fieldNames = structTypeInfo.getAllStructFieldNames(); + List fieldTypeInfos = structTypeInfo + .getAllStructFieldTypeInfos(); + List fieldObjectInspectors = new ArrayList( + fieldTypeInfos.size()); + for (int i = 0; i < fieldTypeInfos.size(); i++) { + fieldObjectInspectors + .add(getLazyBinaryObjectInspectorFromTypeInfo(fieldTypeInfos + .get(i))); } - case LIST: { - ObjectInspector elementObjectInspector = getLazyBinaryObjectInspectorFromTypeInfo( - ((ListTypeInfo)typeInfo).getListElementTypeInfo()); - result = LazyBinaryObjectInspectorFactory.getLazyBinaryListObjectInspector(elementObjectInspector); - break; - } - case MAP: { - MapTypeInfo mapTypeInfo = (MapTypeInfo)typeInfo; - ObjectInspector keyObjectInspector = getLazyBinaryObjectInspectorFromTypeInfo(mapTypeInfo.getMapKeyTypeInfo()); - ObjectInspector valueObjectInspector = getLazyBinaryObjectInspectorFromTypeInfo(mapTypeInfo.getMapValueTypeInfo()); - result = LazyBinaryObjectInspectorFactory.getLazyBinaryMapObjectInspector(keyObjectInspector, valueObjectInspector); - break; - } - case STRUCT: { - StructTypeInfo structTypeInfo = (StructTypeInfo)typeInfo; - List fieldNames = structTypeInfo.getAllStructFieldNames(); - List fieldTypeInfos = structTypeInfo.getAllStructFieldTypeInfos(); - List fieldObjectInspectors = new ArrayList(fieldTypeInfos.size()); - for(int i=0; i columnNames; - List columnTypes; - - TypeInfo rowTypeInfo; + List columnTypes; + + TypeInfo rowTypeInfo; ObjectInspector cachedObjectInspector; - + // The object for storing row data LazyBinaryStruct cachedLazyBinaryStruct; - + /** * Initialize the SerDe with configuration and table information */ @@ -95,39 +95,41 @@ if (columnTypeProperty.length() == 0) { columnTypes = new ArrayList(); } else { - columnTypes = TypeInfoUtils.getTypeInfosFromTypeString(columnTypeProperty); + columnTypes = TypeInfoUtils + .getTypeInfosFromTypeString(columnTypeProperty); } - assert(columnNames.size() == columnTypes.size()); + assert (columnNames.size() == columnTypes.size()); // Create row related objects - rowTypeInfo = TypeInfoFactory.getStructTypeInfo(columnNames, columnTypes); + rowTypeInfo = TypeInfoFactory.getStructTypeInfo(columnNames, columnTypes); // Create the object inspector and the lazy binary struct object - cachedObjectInspector = LazyBinaryUtils.getLazyBinaryObjectInspectorFromTypeInfo(rowTypeInfo); + cachedObjectInspector = LazyBinaryUtils + .getLazyBinaryObjectInspectorFromTypeInfo(rowTypeInfo); cachedLazyBinaryStruct = (LazyBinaryStruct) LazyBinaryFactory - .createLazyBinaryObject(cachedObjectInspector); + .createLazyBinaryObject(cachedObjectInspector); // output debug info - LOG.debug("LazyBinarySerDe initialized with: columnNames=" - + columnNames + " columnTypes=" + columnTypes); + LOG.debug("LazyBinarySerDe initialized with: columnNames=" + columnNames + + " columnTypes=" + columnTypes); } - + /** * Returns the ObjectInspector for the row. */ @Override public ObjectInspector getObjectInspector() throws SerDeException { - return cachedObjectInspector; + return cachedObjectInspector; } /** * Returns the Writable Class after serialization. */ @Override - public Class getSerializedClass() { + public Class getSerializedClass() { return BytesWritable.class; } // The wrapper for byte array ByteArrayRef byteArrayRef; - + /** * Deserialize a table record to a lazybinary struct. */ @@ -137,29 +139,31 @@ byteArrayRef = new ByteArrayRef(); } if (field instanceof BytesWritable) { - BytesWritable b = (BytesWritable)field; - if(b.getSize()==0) + BytesWritable b = (BytesWritable) field; + if (b.getSize() == 0) { return null; + } // For backward-compatibility with hadoop 0.17 byteArrayRef.setData(b.get()); cachedLazyBinaryStruct.init(byteArrayRef, 0, b.getSize()); } else if (field instanceof Text) { - Text t = (Text)field; - if(t.getLength()==0) + Text t = (Text) field; + if (t.getLength() == 0) { return null; + } byteArrayRef.setData(t.getBytes()); cachedLazyBinaryStruct.init(byteArrayRef, 0, t.getLength()); } else { - throw new SerDeException(getClass().toString() + throw new SerDeException(getClass().toString() + ": expects either BytesWritable or Text object!"); } return cachedLazyBinaryStruct; } - + /** * The reusable output buffer and serialize byte buffer. */ - BytesWritable serializeBytesWritable = new BytesWritable(); + BytesWritable serializeBytesWritable = new BytesWritable(); ByteStream.Output serializeByteStream = new ByteStream.Output(); /** @@ -170,293 +174,300 @@ throws SerDeException { // make sure it is a struct record if (objInspector.getCategory() != Category.STRUCT) { - throw new SerDeException(getClass().toString() - + " can only serialize struct types, but we got: " + throw new SerDeException(getClass().toString() + + " can only serialize struct types, but we got: " + objInspector.getTypeName()); } - - serializeByteStream.reset(); + + serializeByteStream.reset(); // serialize the row as a struct - serializeStruct(serializeByteStream, obj, - (StructObjectInspector) objInspector); - // return the serialized bytes - serializeBytesWritable.set(serializeByteStream.getData(), - 0, serializeByteStream.getCount()); + serializeStruct(serializeByteStream, obj, + (StructObjectInspector) objInspector); + // return the serialized bytes + serializeBytesWritable.set(serializeByteStream.getData(), 0, + serializeByteStream.getCount()); return serializeBytesWritable; } boolean nullMapKey = false; - + /** - * Serialize a struct object without writing the byte size. - * This function is shared by both row serialization and - * struct serialization. + * Serialize a struct object without writing the byte size. This function is + * shared by both row serialization and struct serialization. * - * @param byteStream the byte stream storing the serialization data - * @param obj the struct object to serialize - * @param objInspector the struct object inspector + * @param byteStream + * the byte stream storing the serialization data + * @param obj + * the struct object to serialize + * @param objInspector + * the struct object inspector */ - private void serializeStruct(Output byteStream, - Object obj, StructObjectInspector soi) { + private void serializeStruct(Output byteStream, Object obj, + StructObjectInspector soi) { // do nothing for null struct - if(null == obj) + if (null == obj) { return; - /* - * Interleave serializing one null byte and 8 struct fields - * in each round, in order to support data deserialization - * with different table schemas + } + /* + * Interleave serializing one null byte and 8 struct fields in each round, + * in order to support data deserialization with different table schemas */ - List fields = soi.getAllStructFieldRefs(); - int size = fields.size(); + List fields = soi.getAllStructFieldRefs(); + int size = fields.size(); int lasti = 0; byte nullByte = 0; - for (int i=0; i> 8)); - byteStream.write((byte) (v)); - return; - } - case INT: { - IntObjectInspector ioi = (IntObjectInspector)poi; - int v = ioi.get(obj); - LazyBinaryUtils.writeVInt(byteStream, v); - return; - } - case LONG: { - LongObjectInspector loi = (LongObjectInspector)poi; - long v = loi.get(obj); - LazyBinaryUtils.writeVLong(byteStream, v); - return; - } - case FLOAT: { - FloatObjectInspector foi = (FloatObjectInspector)poi; - int v = Float.floatToIntBits(foi.get(obj)); - byteStream.write((byte) (v >> 24)); - byteStream.write((byte) (v >> 16)); - byteStream.write((byte) (v >> 8)); - byteStream.write((byte) (v)); - return; - } - case DOUBLE: { - DoubleObjectInspector doi = (DoubleObjectInspector)poi; - long v = Double.doubleToLongBits(doi.get(obj)); - byteStream.write((byte) (v >> 56)); - byteStream.write((byte) (v >> 48)); - byteStream.write((byte) (v >> 40)); - byteStream.write((byte) (v >> 32)); - byteStream.write((byte) (v >> 24)); - byteStream.write((byte) (v >> 16)); - byteStream.write((byte) (v >> 8)); - byteStream.write((byte) (v)); - return; - } - case STRING: { - StringObjectInspector soi = (StringObjectInspector)poi; - Text t = soi.getPrimitiveWritableObject(obj); - /* write byte size of the string which is a vint */ - int length = t.getLength(); - LazyBinaryUtils.writeVInt(byteStream, length); - /* write string itself */ - byte[] data = t.getBytes(); - byteStream.write(data, 0, length); - return; - } - default: { - throw new RuntimeException("Unrecognized type: " + poi.getPrimitiveCategory()); - } - } + case PRIMITIVE: { + PrimitiveObjectInspector poi = (PrimitiveObjectInspector) objInspector; + switch (poi.getPrimitiveCategory()) { + case VOID: { + return; } - case LIST: { - ListObjectInspector loi = (ListObjectInspector)objInspector; - ObjectInspector eoi = loi.getListElementObjectInspector(); - - // 1/ reserve spaces for the byte size of the list - // which is a integer and takes four bytes - int byteSizeStart = byteStream.getCount(); - byteStream.write((byte) 0); - byteStream.write((byte) 0); - byteStream.write((byte) 0); - byteStream.write((byte) 0); - int listStart = byteStream.getCount(); - - // 2/ write the size of the list as a VInt - int size = loi.getListLength(obj); - LazyBinaryUtils.writeVInt(byteStream, size); - - // 3/ write the null bytes - byte nullByte = 0; - for (int eid = 0; eid < size; eid++) { - // set the bit to 1 if an element is not null - if (null != loi.getListElement(obj, eid)) { - nullByte |= 1 << (eid%8); - } - // store the byte every eight elements or - // if this is the last element - if (7 == eid%8 || eid == size-1) { - byteStream.write(nullByte); - nullByte = 0; - } - } - - // 4/ write element by element from the list - for (int eid = 0; eid < size; eid++) { - serialize(byteStream, loi.getListElement(obj, eid), eoi); - } - - // 5/ update the list byte size - int listEnd = byteStream.getCount(); - int listSize = listEnd - listStart; - byte [] bytes = byteStream.getData(); - bytes[byteSizeStart ] = (byte) (listSize >> 24); - bytes[byteSizeStart + 1] = (byte) (listSize >> 16); - bytes[byteSizeStart + 2] = (byte) (listSize >> 8); - bytes[byteSizeStart + 3] = (byte) (listSize); - + case BOOLEAN: { + boolean v = ((BooleanObjectInspector) poi).get(obj); + byteStream.write((byte) (v ? 1 : 0)); return; - } - case MAP: { - MapObjectInspector moi = (MapObjectInspector)objInspector; - ObjectInspector koi = moi.getMapKeyObjectInspector(); - ObjectInspector voi = moi.getMapValueObjectInspector(); - Map map = moi.getMap(obj); - - // 1/ reserve spaces for the byte size of the map - // which is a integer and takes four bytes - int byteSizeStart = byteStream.getCount(); - byteStream.write((byte) 0); - byteStream.write((byte) 0); - byteStream.write((byte) 0); - byteStream.write((byte) 0); - int mapStart = byteStream.getCount(); - - // 2/ write the size of the map which is a VInt - int size = map.size(); - LazyBinaryUtils.writeVInt(byteStream, size); - - // 3/ write the null bytes - int b = 0; - byte nullByte = 0; - for (Map.Entry entry: map.entrySet()) { - // set the bit to 1 if a key is not null - if (null != entry.getKey()) { - nullByte |= 1 << (b%8); - } else if (!nullMapKey) { - nullMapKey = true; - LOG.warn("Null map key encountered! Ignoring similar problems."); - } - b++; - // set the bit to 1 if a value is not null - if (null != entry.getValue()) { - nullByte |= 1 << (b%8); - } - b++; - // write the byte to stream every 4 key-value pairs - // or if this is the last key-value pair - if (0 == b%8 || b == size*2) { - byteStream.write(nullByte); - nullByte = 0; - } - } - - // 4/ write key-value pairs one by one - for(Map.Entry entry: map.entrySet()) { - serialize(byteStream, entry.getKey(), koi); - serialize(byteStream, entry.getValue(), voi); - } - - // 5/ update the byte size of the map - int mapEnd = byteStream.getCount(); - int mapSize = mapEnd - mapStart; - byte [] bytes = byteStream.getData(); - bytes[byteSizeStart ] = (byte) (mapSize >> 24); - bytes[byteSizeStart + 1] = (byte) (mapSize >> 16); - bytes[byteSizeStart + 2] = (byte) (mapSize >> 8); - bytes[byteSizeStart + 3] = (byte) (mapSize); - + } + case BYTE: { + ByteObjectInspector boi = (ByteObjectInspector) poi; + byte v = boi.get(obj); + byteStream.write(v); return; } - case STRUCT: { - // 1/ reserve spaces for the byte size of the struct - // which is a integer and takes four bytes - int byteSizeStart = byteStream.getCount(); - byteStream.write((byte) 0); - byteStream.write((byte) 0); - byteStream.write((byte) 0); - byteStream.write((byte) 0); - int structStart = byteStream.getCount(); - - // 2/ serialize the struct - serializeStruct(byteStream, obj, (StructObjectInspector) objInspector); - - // 3/ update the byte size of the struct - int structEnd = byteStream.getCount(); - int structSize = structEnd - structStart; - byte [] bytes = byteStream.getData(); - bytes[byteSizeStart ] = (byte) (structSize >> 24); - bytes[byteSizeStart + 1] = (byte) (structSize >> 16); - bytes[byteSizeStart + 2] = (byte) (structSize >> 8); - bytes[byteSizeStart + 3] = (byte) (structSize); - + case SHORT: { + ShortObjectInspector spoi = (ShortObjectInspector) poi; + short v = spoi.get(obj); + byteStream.write((byte) (v >> 8)); + byteStream.write((byte) (v)); return; } + case INT: { + IntObjectInspector ioi = (IntObjectInspector) poi; + int v = ioi.get(obj); + LazyBinaryUtils.writeVInt(byteStream, v); + return; + } + case LONG: { + LongObjectInspector loi = (LongObjectInspector) poi; + long v = loi.get(obj); + LazyBinaryUtils.writeVLong(byteStream, v); + return; + } + case FLOAT: { + FloatObjectInspector foi = (FloatObjectInspector) poi; + int v = Float.floatToIntBits(foi.get(obj)); + byteStream.write((byte) (v >> 24)); + byteStream.write((byte) (v >> 16)); + byteStream.write((byte) (v >> 8)); + byteStream.write((byte) (v)); + return; + } + case DOUBLE: { + DoubleObjectInspector doi = (DoubleObjectInspector) poi; + long v = Double.doubleToLongBits(doi.get(obj)); + byteStream.write((byte) (v >> 56)); + byteStream.write((byte) (v >> 48)); + byteStream.write((byte) (v >> 40)); + byteStream.write((byte) (v >> 32)); + byteStream.write((byte) (v >> 24)); + byteStream.write((byte) (v >> 16)); + byteStream.write((byte) (v >> 8)); + byteStream.write((byte) (v)); + return; + } + case STRING: { + StringObjectInspector soi = (StringObjectInspector) poi; + Text t = soi.getPrimitiveWritableObject(obj); + /* write byte size of the string which is a vint */ + int length = t.getLength(); + LazyBinaryUtils.writeVInt(byteStream, length); + /* write string itself */ + byte[] data = t.getBytes(); + byteStream.write(data, 0, length); + return; + } default: { - throw new RuntimeException("Unrecognized type: " + objInspector.getCategory()); + throw new RuntimeException("Unrecognized type: " + + poi.getPrimitiveCategory()); } + } } + case LIST: { + ListObjectInspector loi = (ListObjectInspector) objInspector; + ObjectInspector eoi = loi.getListElementObjectInspector(); + + // 1/ reserve spaces for the byte size of the list + // which is a integer and takes four bytes + int byteSizeStart = byteStream.getCount(); + byteStream.write((byte) 0); + byteStream.write((byte) 0); + byteStream.write((byte) 0); + byteStream.write((byte) 0); + int listStart = byteStream.getCount(); + + // 2/ write the size of the list as a VInt + int size = loi.getListLength(obj); + LazyBinaryUtils.writeVInt(byteStream, size); + + // 3/ write the null bytes + byte nullByte = 0; + for (int eid = 0; eid < size; eid++) { + // set the bit to 1 if an element is not null + if (null != loi.getListElement(obj, eid)) { + nullByte |= 1 << (eid % 8); + } + // store the byte every eight elements or + // if this is the last element + if (7 == eid % 8 || eid == size - 1) { + byteStream.write(nullByte); + nullByte = 0; + } + } + + // 4/ write element by element from the list + for (int eid = 0; eid < size; eid++) { + serialize(byteStream, loi.getListElement(obj, eid), eoi); + } + + // 5/ update the list byte size + int listEnd = byteStream.getCount(); + int listSize = listEnd - listStart; + byte[] bytes = byteStream.getData(); + bytes[byteSizeStart] = (byte) (listSize >> 24); + bytes[byteSizeStart + 1] = (byte) (listSize >> 16); + bytes[byteSizeStart + 2] = (byte) (listSize >> 8); + bytes[byteSizeStart + 3] = (byte) (listSize); + + return; + } + case MAP: { + MapObjectInspector moi = (MapObjectInspector) objInspector; + ObjectInspector koi = moi.getMapKeyObjectInspector(); + ObjectInspector voi = moi.getMapValueObjectInspector(); + Map map = moi.getMap(obj); + + // 1/ reserve spaces for the byte size of the map + // which is a integer and takes four bytes + int byteSizeStart = byteStream.getCount(); + byteStream.write((byte) 0); + byteStream.write((byte) 0); + byteStream.write((byte) 0); + byteStream.write((byte) 0); + int mapStart = byteStream.getCount(); + + // 2/ write the size of the map which is a VInt + int size = map.size(); + LazyBinaryUtils.writeVInt(byteStream, size); + + // 3/ write the null bytes + int b = 0; + byte nullByte = 0; + for (Map.Entry entry : map.entrySet()) { + // set the bit to 1 if a key is not null + if (null != entry.getKey()) { + nullByte |= 1 << (b % 8); + } else if (!nullMapKey) { + nullMapKey = true; + LOG.warn("Null map key encountered! Ignoring similar problems."); + } + b++; + // set the bit to 1 if a value is not null + if (null != entry.getValue()) { + nullByte |= 1 << (b % 8); + } + b++; + // write the byte to stream every 4 key-value pairs + // or if this is the last key-value pair + if (0 == b % 8 || b == size * 2) { + byteStream.write(nullByte); + nullByte = 0; + } + } + + // 4/ write key-value pairs one by one + for (Map.Entry entry : map.entrySet()) { + serialize(byteStream, entry.getKey(), koi); + serialize(byteStream, entry.getValue(), voi); + } + + // 5/ update the byte size of the map + int mapEnd = byteStream.getCount(); + int mapSize = mapEnd - mapStart; + byte[] bytes = byteStream.getData(); + bytes[byteSizeStart] = (byte) (mapSize >> 24); + bytes[byteSizeStart + 1] = (byte) (mapSize >> 16); + bytes[byteSizeStart + 2] = (byte) (mapSize >> 8); + bytes[byteSizeStart + 3] = (byte) (mapSize); + + return; + } + case STRUCT: { + // 1/ reserve spaces for the byte size of the struct + // which is a integer and takes four bytes + int byteSizeStart = byteStream.getCount(); + byteStream.write((byte) 0); + byteStream.write((byte) 0); + byteStream.write((byte) 0); + byteStream.write((byte) 0); + int structStart = byteStream.getCount(); + + // 2/ serialize the struct + serializeStruct(byteStream, obj, (StructObjectInspector) objInspector); + + // 3/ update the byte size of the struct + int structEnd = byteStream.getCount(); + int structSize = structEnd - structStart; + byte[] bytes = byteStream.getData(); + bytes[byteSizeStart] = (byte) (structSize >> 24); + bytes[byteSizeStart + 1] = (byte) (structSize >> 16); + bytes[byteSizeStart + 2] = (byte) (structSize >> 8); + bytes[byteSizeStart + 3] = (byte) (structSize); + + return; + } + default: { + throw new RuntimeException("Unrecognized type: " + + objInspector.getCategory()); + } + } } } Index: serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryByte.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryByte.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryByte.java (working copy) @@ -17,14 +17,15 @@ */ package org.apache.hadoop.hive.serde2.lazybinary; +import org.apache.hadoop.hive.serde2.io.ByteWritable; import org.apache.hadoop.hive.serde2.lazy.ByteArrayRef; import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableByteObjectInspector; -import org.apache.hadoop.hive.serde2.io.ByteWritable; /** * LazyBinaryObject for byte which takes one byte */ -public class LazyBinaryByte extends LazyBinaryPrimitive { +public class LazyBinaryByte extends + LazyBinaryPrimitive { LazyBinaryByte(WritableByteObjectInspector oi) { super(oi); @@ -35,10 +36,10 @@ super(copy); data = new ByteWritable(copy.data.get()); } - + @Override public void init(ByteArrayRef bytes, int start, int length) { - assert(1 == length); + assert (1 == length); data.set(bytes.getData()[start]); } } Index: serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryFloat.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryFloat.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryFloat.java (working copy) @@ -24,13 +24,14 @@ /** * LazyBinaryObject for float which takes four bytes. */ -public class LazyBinaryFloat extends LazyBinaryPrimitive { +public class LazyBinaryFloat extends + LazyBinaryPrimitive { LazyBinaryFloat(WritableFloatObjectInspector oi) { super(oi); data = new FloatWritable(); } - + LazyBinaryFloat(LazyBinaryFloat copy) { super(copy); data = new FloatWritable(copy.data.get()); @@ -39,6 +40,7 @@ @Override public void init(ByteArrayRef bytes, int start, int length) { assert (4 == length); - data.set(Float.intBitsToFloat(LazyBinaryUtils.byteArrayToInt(bytes.getData(), start))); + data.set(Float.intBitsToFloat(LazyBinaryUtils.byteArrayToInt(bytes + .getData(), start))); } } Index: serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryObject.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryObject.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryObject.java (working copy) @@ -21,46 +21,51 @@ import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; /** - * LazyBinaryObject stores an object in a binary format in a byte[]. - * For example, a double takes four bytes. + * LazyBinaryObject stores an object in a binary format in a byte[]. For + * example, a double takes four bytes. * * A LazyBinaryObject can represent any primitive object or hierarchical object * like string, list, map or struct. */ public abstract class LazyBinaryObject { - + OI oi; - + /** * Create a LazyBinaryObject. - * @param oi Derived classes can access meta information about this Lazy - * Binary Object (e.g, length, null-bits) from it. + * + * @param oi + * Derived classes can access meta information about this Lazy Binary + * Object (e.g, length, null-bits) from it. */ protected LazyBinaryObject(OI oi) { this.oi = oi; } - + /** - * Set the data for this LazyBinaryObject. - * We take ByteArrayRef instead of byte[] so that we will be able to drop - * the reference to byte[] by a single assignment. - * The ByteArrayRef object can be reused across multiple rows. + * Set the data for this LazyBinaryObject. We take ByteArrayRef instead of + * byte[] so that we will be able to drop the reference to byte[] by a single + * assignment. The ByteArrayRef object can be reused across multiple rows. * * Never call this function if the object represent a null!!! * - * @param bytes The wrapper of the byte[]. - * @param start The start position inside the bytes. - * @param length The length of the data, starting from "start" + * @param bytes + * The wrapper of the byte[]. + * @param start + * The start position inside the bytes. + * @param length + * The length of the data, starting from "start" * @see ByteArrayRef */ public abstract void init(ByteArrayRef bytes, int start, int length); - + /** - * If the LazyBinaryObject is a primitive Object, then deserialize it and return - * the actual primitive Object. - * Otherwise (string, list, map, struct), return this. + * If the LazyBinaryObject is a primitive Object, then deserialize it and + * return the actual primitive Object. Otherwise (string, list, map, struct), + * return this. */ public abstract Object getObject(); - + + @Override public abstract int hashCode(); } Index: serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryString.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryString.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryString.java (working copy) @@ -17,34 +17,34 @@ */ package org.apache.hadoop.hive.serde2.lazybinary; -import org.apache.hadoop.io.Text; import org.apache.hadoop.hive.serde2.lazy.ByteArrayRef; import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableStringObjectInspector; +import org.apache.hadoop.io.Text; /** - * The serialization of LazyBinaryString is very simple: - * start A end - * bytes[] -> |---------------------------------| + * The serialization of LazyBinaryString is very simple: start A end bytes[] -> + * |---------------------------------| * - * Section A is just an array of bytes which are exactly - * the Text contained in this object. + * Section A is just an array of bytes which are exactly the Text contained in + * this object. * */ -public class LazyBinaryString extends LazyBinaryPrimitive { +public class LazyBinaryString extends + LazyBinaryPrimitive { LazyBinaryString(WritableStringObjectInspector OI) { super(OI); data = new Text(); } - + public LazyBinaryString(LazyBinaryString copy) { super(copy); data = new Text(copy.data); - } + } @Override public void init(ByteArrayRef bytes, int start, int length) { - assert(length > -1); + assert (length > -1); data.set(bytes.getData(), start, length); } } Index: serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryFactory.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryFactory.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryFactory.java (working copy) @@ -35,39 +35,40 @@ public class LazyBinaryFactory { /** - * Create a lazy binary primitive class given the type name. + * Create a lazy binary primitive class given the type name. */ - public static LazyBinaryPrimitive createLazyBinaryPrimitiveClass(PrimitiveObjectInspector oi) { + public static LazyBinaryPrimitive createLazyBinaryPrimitiveClass( + PrimitiveObjectInspector oi) { PrimitiveCategory p = oi.getPrimitiveCategory(); - switch(p) { - case BOOLEAN: { - return new LazyBinaryBoolean((WritableBooleanObjectInspector)oi); - } - case BYTE: { - return new LazyBinaryByte((WritableByteObjectInspector)oi); - } - case SHORT: { - return new LazyBinaryShort((WritableShortObjectInspector)oi); - } - case INT: { - return new LazyBinaryInteger((WritableIntObjectInspector)oi); - } - case LONG: { - return new LazyBinaryLong((WritableLongObjectInspector)oi); - } - case FLOAT: { - return new LazyBinaryFloat((WritableFloatObjectInspector)oi); - } - case DOUBLE: { - return new LazyBinaryDouble((WritableDoubleObjectInspector)oi); - } - case STRING: { - return new LazyBinaryString((WritableStringObjectInspector)oi); - } - default: { - throw new RuntimeException("Internal error: no LazyBinaryObject for " + p); - } + switch (p) { + case BOOLEAN: { + return new LazyBinaryBoolean((WritableBooleanObjectInspector) oi); } + case BYTE: { + return new LazyBinaryByte((WritableByteObjectInspector) oi); + } + case SHORT: { + return new LazyBinaryShort((WritableShortObjectInspector) oi); + } + case INT: { + return new LazyBinaryInteger((WritableIntObjectInspector) oi); + } + case LONG: { + return new LazyBinaryLong((WritableLongObjectInspector) oi); + } + case FLOAT: { + return new LazyBinaryFloat((WritableFloatObjectInspector) oi); + } + case DOUBLE: { + return new LazyBinaryDouble((WritableDoubleObjectInspector) oi); + } + case STRING: { + return new LazyBinaryString((WritableStringObjectInspector) oi); + } + default: { + throw new RuntimeException("Internal error: no LazyBinaryObject for " + p); + } + } } /** @@ -75,17 +76,17 @@ */ public static LazyBinaryObject createLazyBinaryObject(ObjectInspector oi) { ObjectInspector.Category c = oi.getCategory(); - switch(c) { + switch (c) { case PRIMITIVE: - return createLazyBinaryPrimitiveClass((PrimitiveObjectInspector)oi); + return createLazyBinaryPrimitiveClass((PrimitiveObjectInspector) oi); case MAP: - return new LazyBinaryMap((LazyBinaryMapObjectInspector)oi); - case LIST: - return new LazyBinaryArray((LazyBinaryListObjectInspector)oi); + return new LazyBinaryMap((LazyBinaryMapObjectInspector) oi); + case LIST: + return new LazyBinaryArray((LazyBinaryListObjectInspector) oi); case STRUCT: - return new LazyBinaryStruct((LazyBinaryStructObjectInspector)oi); + return new LazyBinaryStruct((LazyBinaryStructObjectInspector) oi); } throw new RuntimeException("Hive LazyBinarySerDe Internal error."); - } + } } Index: serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/objectinspector/LazyBinaryMapObjectInspector.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/objectinspector/LazyBinaryMapObjectInspector.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/objectinspector/LazyBinaryMapObjectInspector.java (working copy) @@ -25,6 +25,7 @@ /** * ObjectInspector for LazyBinaryMap + * * @see LazyBinaryMap */ public class LazyBinaryMapObjectInspector extends StandardMapObjectInspector { @@ -39,7 +40,7 @@ if (data == null) { return null; } - return ((LazyBinaryMap)data).getMap(); + return ((LazyBinaryMap) data).getMap(); } @Override @@ -47,7 +48,7 @@ if (data == null) { return -1; } - return ((LazyBinaryMap)data).getMapSize(); + return ((LazyBinaryMap) data).getMapSize(); } @Override @@ -55,6 +56,6 @@ if (data == null) { return -1; } - return ((LazyBinaryMap)data).getMapValueElement(key); + return ((LazyBinaryMap) data).getMapValueElement(key); } } Index: serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/objectinspector/LazyBinaryStructObjectInspector.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/objectinspector/LazyBinaryStructObjectInspector.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/objectinspector/LazyBinaryStructObjectInspector.java (working copy) @@ -26,29 +26,32 @@ /** * ObjectInspector for LazyBinaryStruct + * * @see LazyBinaryStruct */ -public class LazyBinaryStructObjectInspector extends StandardStructObjectInspector { +public class LazyBinaryStructObjectInspector extends + StandardStructObjectInspector { - protected LazyBinaryStructObjectInspector(List structFieldNames, List structFieldObjectInspectors) { + protected LazyBinaryStructObjectInspector(List structFieldNames, + List structFieldObjectInspectors) { super(structFieldNames, structFieldObjectInspectors); } - + protected LazyBinaryStructObjectInspector(List fields) { - super(fields); + super(fields); } - + @Override public Object getStructFieldData(Object data, StructField fieldRef) { if (data == null) { return null; } - LazyBinaryStruct struct = (LazyBinaryStruct)data; + LazyBinaryStruct struct = (LazyBinaryStruct) data; MyField f = (MyField) fieldRef; - + int fieldID = f.getFieldID(); - assert(fieldID >= 0 && fieldID < fields.size()); - + assert (fieldID >= 0 && fieldID < fields.size()); + return struct.getField(fieldID); } @@ -57,7 +60,7 @@ if (data == null) { return null; } - LazyBinaryStruct struct = (LazyBinaryStruct)data; + LazyBinaryStruct struct = (LazyBinaryStruct) data; return struct.getFieldsAsList(); } } Index: serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/objectinspector/LazyBinaryObjectInspectorFactory.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/objectinspector/LazyBinaryObjectInspectorFactory.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/objectinspector/LazyBinaryObjectInspectorFactory.java (working copy) @@ -7,55 +7,62 @@ import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; /** - * ObjectInspectorFactory is the primary way to create new ObjectInspector instances. + * ObjectInspectorFactory is the primary way to create new ObjectInspector + * instances. * - * SerDe classes should call the static functions in this library to create an ObjectInspector - * to return to the caller of SerDe2.getObjectInspector(). + * SerDe classes should call the static functions in this library to create an + * ObjectInspector to return to the caller of SerDe2.getObjectInspector(). * - * The reason of having caches here is that ObjectInspectors do not have an internal - * state - so ObjectInspectors with the same construction parameters should - * result in exactly the same ObjectInspector. + * The reason of having caches here is that ObjectInspectors do not have an + * internal state - so ObjectInspectors with the same construction parameters + * should result in exactly the same ObjectInspector. */ public class LazyBinaryObjectInspectorFactory { - static HashMap, LazyBinaryStructObjectInspector> cachedLazyBinaryStructObjectInspector = - new HashMap, LazyBinaryStructObjectInspector>(); - public static LazyBinaryStructObjectInspector getLazyBinaryStructObjectInspector(List structFieldNames, + static HashMap, LazyBinaryStructObjectInspector> cachedLazyBinaryStructObjectInspector = new HashMap, LazyBinaryStructObjectInspector>(); + + public static LazyBinaryStructObjectInspector getLazyBinaryStructObjectInspector( + List structFieldNames, List structFieldObjectInspectors) { ArrayList signature = new ArrayList(); signature.add(structFieldNames); signature.add(structFieldObjectInspectors); - LazyBinaryStructObjectInspector result = cachedLazyBinaryStructObjectInspector.get(signature); + LazyBinaryStructObjectInspector result = cachedLazyBinaryStructObjectInspector + .get(signature); if (result == null) { - result = new LazyBinaryStructObjectInspector(structFieldNames, structFieldObjectInspectors); + result = new LazyBinaryStructObjectInspector(structFieldNames, + structFieldObjectInspectors); cachedLazyBinaryStructObjectInspector.put(signature, result); } return result; } - - static HashMap, LazyBinaryListObjectInspector> cachedLazyBinaryListObjectInspector = - new HashMap, LazyBinaryListObjectInspector>(); - public static LazyBinaryListObjectInspector getLazyBinaryListObjectInspector( + + static HashMap, LazyBinaryListObjectInspector> cachedLazyBinaryListObjectInspector = new HashMap, LazyBinaryListObjectInspector>(); + + public static LazyBinaryListObjectInspector getLazyBinaryListObjectInspector( ObjectInspector listElementObjectInspector) { ArrayList signature = new ArrayList(); signature.add(listElementObjectInspector); - LazyBinaryListObjectInspector result = cachedLazyBinaryListObjectInspector.get(signature); + LazyBinaryListObjectInspector result = cachedLazyBinaryListObjectInspector + .get(signature); if (result == null) { result = new LazyBinaryListObjectInspector(listElementObjectInspector); cachedLazyBinaryListObjectInspector.put(signature, result); } return result; } - - static HashMap, LazyBinaryMapObjectInspector> cachedLazyBinaryMapObjectInspector = - new HashMap, LazyBinaryMapObjectInspector>(); - public static LazyBinaryMapObjectInspector getLazyBinaryMapObjectInspector( - ObjectInspector mapKeyObjectInspector, ObjectInspector mapValueObjectInspector) { + + static HashMap, LazyBinaryMapObjectInspector> cachedLazyBinaryMapObjectInspector = new HashMap, LazyBinaryMapObjectInspector>(); + + public static LazyBinaryMapObjectInspector getLazyBinaryMapObjectInspector( + ObjectInspector mapKeyObjectInspector, + ObjectInspector mapValueObjectInspector) { ArrayList signature = new ArrayList(); signature.add(mapKeyObjectInspector); signature.add(mapValueObjectInspector); - LazyBinaryMapObjectInspector result = cachedLazyBinaryMapObjectInspector.get(signature); + LazyBinaryMapObjectInspector result = cachedLazyBinaryMapObjectInspector + .get(signature); if (result == null) { result = new LazyBinaryMapObjectInspector(mapKeyObjectInspector, mapValueObjectInspector); Index: serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/objectinspector/LazyBinaryListObjectInspector.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/objectinspector/LazyBinaryListObjectInspector.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/objectinspector/LazyBinaryListObjectInspector.java (working copy) @@ -20,8 +20,8 @@ import java.util.List; import org.apache.hadoop.hive.serde2.lazybinary.LazyBinaryArray; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.StandardListObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; /** * ObjectInspector for LazyBinaryList Index: serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryPrimitive.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryPrimitive.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryPrimitive.java (working copy) @@ -21,32 +21,33 @@ import org.apache.hadoop.io.Writable; /** - * Defines a LazyBinaryPrimitive. + * Defines a LazyBinaryPrimitive. * - * data will be initialized to corresponding types in - * different LazyBinary primitive classes. For example, data will - * be a BooleanWritable in the LazyBinaryBoolean class. + * data will be initialized to corresponding types in different LazyBinary + * primitive classes. For example, data will be a BooleanWritable in the + * LazyBinaryBoolean class. * - * There is no null flag any more, - * + * There is no null flag any more, + * */ -public abstract class LazyBinaryPrimitive extends LazyBinaryObject { +public abstract class LazyBinaryPrimitive + extends LazyBinaryObject { LazyBinaryPrimitive(OI oi) { super(oi); } - + LazyBinaryPrimitive(LazyBinaryPrimitive copy) { super(copy.oi); } T data; - + /** - * Returns the primitive object represented by this LazyBinaryObject. - * This is useful because it can make sure we have "null" for null objects. + * Returns the primitive object represented by this LazyBinaryObject. This is + * useful because it can make sure we have "null" for null objects. */ + @Override public Object getObject() { return data; } @@ -54,12 +55,14 @@ public T getWritableObject() { return data; } - + + @Override public String toString() { return data.toString(); } - - public int hashCode(){ + + @Override + public int hashCode() { return data == null ? 0 : data.hashCode(); } } Index: serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryDouble.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryDouble.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryDouble.java (working copy) @@ -17,21 +17,21 @@ */ package org.apache.hadoop.hive.serde2.lazybinary; - +import org.apache.hadoop.hive.serde2.io.DoubleWritable; import org.apache.hadoop.hive.serde2.lazy.ByteArrayRef; import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableDoubleObjectInspector; -import org.apache.hadoop.hive.serde2.io.DoubleWritable; /** * LazyBinaryObject for double which takes eight bytes */ -public class LazyBinaryDouble extends LazyBinaryPrimitive { +public class LazyBinaryDouble extends + LazyBinaryPrimitive { LazyBinaryDouble(WritableDoubleObjectInspector oi) { super(oi); data = new DoubleWritable(); } - + LazyBinaryDouble(LazyBinaryDouble copy) { super(copy); data = new DoubleWritable(copy.data.get()); @@ -39,7 +39,8 @@ @Override public void init(ByteArrayRef bytes, int start, int length) { - assert(8 == length); - data.set(Double.longBitsToDouble(LazyBinaryUtils.byteArrayToLong(bytes.getData(), start))); + assert (8 == length); + data.set(Double.longBitsToDouble(LazyBinaryUtils.byteArrayToLong(bytes + .getData(), start))); } } Index: serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryMap.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryMap.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryMap.java (working copy) @@ -24,8 +24,6 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hive.serde2.lazy.ByteArrayRef; -import org.apache.hadoop.hive.serde2.lazy.LazyObject; -import org.apache.hadoop.hive.serde2.lazy.LazyPrimitive; import org.apache.hadoop.hive.serde2.lazybinary.LazyBinaryUtils.RecordInfo; import org.apache.hadoop.hive.serde2.lazybinary.LazyBinaryUtils.VInt; import org.apache.hadoop.hive.serde2.lazybinary.objectinspector.LazyBinaryMapObjectInspector; @@ -33,22 +31,22 @@ import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; /** - * LazyBinaryMap is serialized as follows: - * start A b c b c b c end - * bytes[] -> |--------|---|---|---|---| ... |---|---| + * LazyBinaryMap is serialized as follows: start A b c b c b c end bytes[] -> + * |--------|---|---|---|---| ... |---|---| * - * Section A is the null-bytes. Suppose the map has - * N key-value pairs, then there are (N*2+7)/8 bytes used as null-bytes. - * Each bit corresponds to a key or a value and it indicates whether - * that key or value is null (0) or not null (1). + * Section A is the null-bytes. Suppose the map has N key-value pairs, then + * there are (N*2+7)/8 bytes used as null-bytes. Each bit corresponds to a key + * or a value and it indicates whether that key or value is null (0) or not null + * (1). * - * After A, all the bytes are actual serialized data of the map, - * which are key-value pairs. b represent the keys and c represent - * the values. Each of them is again a LazyBinaryObject. - * + * After A, all the bytes are actual serialized data of the map, which are + * key-value pairs. b represent the keys and c represent the values. Each of + * them is again a LazyBinaryObject. + * */ -public class LazyBinaryMap extends LazyBinaryNonPrimitive { +public class LazyBinaryMap extends + LazyBinaryNonPrimitive { private static Log LOG = LogFactory.getLog(LazyBinaryMap.class.getName()); @@ -56,17 +54,16 @@ * Whether the data is already parsed or not. */ boolean parsed; - + /** - * The size of the map. - * Only valid when the data is parsed. - * -1 when the map is NULL. + * The size of the map. Only valid when the data is parsed. -1 when the map is + * NULL. */ int mapSize = 0; - + /** - * The beginning position and length of key[i] and value[i]. - * Only valid when the data is parsed. + * The beginning position and length of key[i] and value[i]. Only valid when + * the data is parsed. */ int[] keyStart; int[] keyLength; @@ -79,30 +76,30 @@ boolean[] valueInited; /** - * Whether valueObjects[i]/keyObjects[i] is null or not - * This could not be inferred from the length of the object. - * In particular, a 0-length string is not null. + * Whether valueObjects[i]/keyObjects[i] is null or not This could not be + * inferred from the length of the object. In particular, a 0-length string is + * not null. */ boolean[] keyIsNull; boolean[] valueIsNull; - + /** * The keys are stored in an array of LazyPrimitives. */ - LazyBinaryPrimitive[] keyObjects; + LazyBinaryPrimitive[] keyObjects; /** - * The values are stored in an array of LazyObjects. - * value[index] will start from KeyEnd[index] + 1, - * and ends before KeyStart[index+1] - 1. + * The values are stored in an array of LazyObjects. value[index] will start + * from KeyEnd[index] + 1, and ends before KeyStart[index+1] - 1. */ LazyBinaryObject[] valueObjects; - + protected LazyBinaryMap(LazyBinaryMapObjectInspector oi) { super(oi); } - + /** * Set the row data for this LazyBinaryMap. + * * @see LazyBinaryObject#init(ByteArrayRef, int, int) */ @Override @@ -112,66 +109,63 @@ } /** - * Adjust the size of arrays: - * keyStart, keyLength - * valueStart, valueLength - * keyInited, keyIsNull - * valueInited, valueIsNull + * Adjust the size of arrays: keyStart, keyLength valueStart, valueLength + * keyInited, keyIsNull valueInited, valueIsNull */ protected void adjustArraySize(int newSize) { if (keyStart == null || keyStart.length < newSize) { - keyStart = new int[newSize]; - keyLength = new int[newSize]; - valueStart = new int[newSize]; - valueLength = new int[newSize]; - keyInited = new boolean[newSize]; - keyIsNull = new boolean[newSize]; - valueInited = new boolean[newSize]; - valueIsNull = new boolean[newSize]; - keyObjects = new LazyBinaryPrimitive[newSize]; + keyStart = new int[newSize]; + keyLength = new int[newSize]; + valueStart = new int[newSize]; + valueLength = new int[newSize]; + keyInited = new boolean[newSize]; + keyIsNull = new boolean[newSize]; + valueInited = new boolean[newSize]; + valueIsNull = new boolean[newSize]; + keyObjects = new LazyBinaryPrimitive[newSize]; valueObjects = new LazyBinaryObject[newSize]; } - } - + } + boolean nullMapKey = false; VInt vInt = new LazyBinaryUtils.VInt(); - RecordInfo recordInfo = new LazyBinaryUtils.RecordInfo(); + RecordInfo recordInfo = new LazyBinaryUtils.RecordInfo(); /** - * Parse the byte[] and fill keyStart, keyLength, keyIsNull - * valueStart, valueLength and valueIsNull + * Parse the byte[] and fill keyStart, keyLength, keyIsNull valueStart, + * valueLength and valueIsNull */ - private void parse() { - + private void parse() { + byte[] bytes = this.bytes.getData(); - + // get the VInt that represents the map size - LazyBinaryUtils.readVInt(bytes, start, vInt); + LazyBinaryUtils.readVInt(bytes, start, vInt); mapSize = vInt.value; - if(0 == mapSize) { + if (0 == mapSize) { parsed = true; return; } - + // adjust arrays adjustArraySize(mapSize); // find out the null-bytes - int mapByteStart = start + vInt.length; - int nullByteCur = mapByteStart; - int nullByteEnd = mapByteStart + (mapSize*2+7) / 8; + int mapByteStart = start + vInt.length; + int nullByteCur = mapByteStart; + int nullByteEnd = mapByteStart + (mapSize * 2 + 7) / 8; int lastElementByteEnd = nullByteEnd; // parsing the keys and values one by one - for (int i=0; i lazyKeyI = uncheckedGetKey(i); - if (lazyKeyI == null) continue; - // getWritableObject() will convert LazyPrimitive to actual primitive writable objects. + for (int i = 0; i < mapSize; i++) { + LazyBinaryPrimitive lazyKeyI = uncheckedGetKey(i); + if (lazyKeyI == null) { + continue; + } + // getWritableObject() will convert LazyPrimitive to actual primitive + // writable objects. Object keyI = lazyKeyI.getWritableObject(); - if (keyI == null) continue; + if (keyI == null) { + continue; + } if (keyI.equals(key)) { // Got a match, return the value LazyBinaryObject v = uncheckedGetValue(i); return v == null ? v : v.getObject(); } - } + } return null; } - /** * Get the key object with the index without checking parsed. - * @param index The index into the array starting from 0 + * + * @param index + * The index into the array starting from 0 */ - private LazyBinaryPrimitive uncheckedGetKey(int index) { + private LazyBinaryPrimitive uncheckedGetKey(int index) { if (keyIsNull[index]) { return null; } @@ -266,24 +271,25 @@ keyInited[index] = true; if (keyObjects[index] == null) { // Keys are always primitive - keyObjects[index] = LazyBinaryFactory.createLazyBinaryPrimitiveClass( - (PrimitiveObjectInspector)((MapObjectInspector)oi).getMapKeyObjectInspector()); + keyObjects[index] = LazyBinaryFactory + .createLazyBinaryPrimitiveClass((PrimitiveObjectInspector) ((MapObjectInspector) oi) + .getMapKeyObjectInspector()); } keyObjects[index].init(bytes, keyStart[index], keyLength[index]); } return keyObjects[index]; } - + /** - * cachedMap is reused for different calls to getMap(). - * But each LazyBinaryMap has a separate cachedMap so we won't overwrite the - * data by accident. + * cachedMap is reused for different calls to getMap(). But each LazyBinaryMap + * has a separate cachedMap so we won't overwrite the data by accident. */ LinkedHashMap cachedMap; - + /** - * Return the map object representing this LazyBinaryMap. - * Note that the keyObjects will be Writable primitive objects. + * Return the map object representing this LazyBinaryMap. Note that the + * keyObjects will be Writable primitive objects. + * * @return the map object */ public Map getMap() { @@ -296,11 +302,13 @@ } else { cachedMap.clear(); } - + // go through each element of the map for (int i = 0; i < mapSize; i++) { - LazyBinaryPrimitive lazyKey = uncheckedGetKey(i); - if (lazyKey == null) continue; + LazyBinaryPrimitive lazyKey = uncheckedGetKey(i); + if (lazyKey == null) { + continue; + } Object key = lazyKey.getObject(); // do not overwrite if there are duplicate keys if (key != null && !cachedMap.containsKey(key)) { @@ -311,15 +319,16 @@ } return cachedMap; } - + /** * Get the size of the map represented by this LazyBinaryMap. - * @return The size of the map + * + * @return The size of the map */ public int getMapSize() { if (!parsed) { parse(); } return mapSize; - } + } } \ No newline at end of file Index: serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryArray.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryArray.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryArray.java (working copy) @@ -30,68 +30,67 @@ import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; /** - * LazyBinaryArray is serialized as follows: - * start A b b b b b b end - * bytes[] -> |--------|---|---|---|---| ... |---|---| + * LazyBinaryArray is serialized as follows: start A b b b b b b end bytes[] -> + * |--------|---|---|---|---| ... |---|---| * - * Section A is the null-bytes. Suppose the list has - * N elements, then there are (N+7)/8 bytes used as null-bytes. - * Each bit corresponds to an element and it indicates whether - * that element is null (0) or not null (1). + * Section A is the null-bytes. Suppose the list has N elements, then there are + * (N+7)/8 bytes used as null-bytes. Each bit corresponds to an element and it + * indicates whether that element is null (0) or not null (1). * - * After A, all b(s) represent the elements of the list. - * Each of them is again a LazyBinaryObject. - * + * After A, all b(s) represent the elements of the list. Each of them is again a + * LazyBinaryObject. + * */ -public class LazyBinaryArray extends LazyBinaryNonPrimitive { - +public class LazyBinaryArray extends + LazyBinaryNonPrimitive { + /** * Whether the data is already parsed or not. */ boolean parsed = false; /** - * The length of the array. - * Only valid when the data is parsed. + * The length of the array. Only valid when the data is parsed. */ int arraySize = 0; - + /** - * The start positions and lengths of array elements. - * Only valid when the data is parsed. + * The start positions and lengths of array elements. Only valid when the data + * is parsed. */ int[] elementStart; int[] elementLength; - + /** * Whether an element is initialized or not */ boolean[] elementInited; /** - * Whether an element is null or not. - * Because length is 0 does not means the field is null. - * In particular, a 0-length string is not null. + * Whether an element is null or not. Because length is 0 does not means the + * field is null. In particular, a 0-length string is not null. */ boolean[] elementIsNull; /** - * The elements of the array. Note that we call - * arrayElements[i].init(bytes, begin, length) - * only when that element is accessed. + * The elements of the array. Note that we call arrayElements[i].init(bytes, + * begin, length) only when that element is accessed. */ LazyBinaryObject[] arrayElements; /** * Construct a LazyBinaryArray object with the ObjectInspector. - * @param oi the oi representing the type of this LazyBinaryArray + * + * @param oi + * the oi representing the type of this LazyBinaryArray */ protected LazyBinaryArray(LazyBinaryListObjectInspector oi) { super(oi); } - + /** * Set the row data for this LazyBinaryArray. + * * @see LazyObject#init(ByteArrayRef, int, int) */ @Override @@ -99,13 +98,13 @@ super.init(bytes, start, length); parsed = false; } - + /** - * Enlarge the size of arrays storing information for the elements inside - * the array. + * Enlarge the size of arrays storing information for the elements inside the + * array. */ private void adjustArraySize(int newSize) { - if (elementStart == null || elementStart.length < newSize) { + if (elementStart == null || elementStart.length < newSize) { elementStart = new int[newSize]; elementLength = new int[newSize]; elementInited = new boolean[newSize]; @@ -113,124 +112,127 @@ arrayElements = new LazyBinaryObject[newSize]; } } - + VInt vInt = new LazyBinaryUtils.VInt(); - RecordInfo recordInfo = new LazyBinaryUtils.RecordInfo(); + RecordInfo recordInfo = new LazyBinaryUtils.RecordInfo(); + /** - * Parse the bytes and fill elementStart, elementLength, - * elementInited and elementIsNull. + * Parse the bytes and fill elementStart, elementLength, elementInited and + * elementIsNull. */ private void parse() { - + byte[] bytes = this.bytes.getData(); - + // get the vlong that represents the map size - LazyBinaryUtils.readVInt(bytes, start, vInt); + LazyBinaryUtils.readVInt(bytes, start, vInt); arraySize = vInt.value; - if(0 == arraySize) { + if (0 == arraySize) { parsed = true; return; } - + // adjust arrays adjustArraySize(arraySize); // find out the null-bytes - int arryByteStart = start + vInt.length; - int nullByteCur = arryByteStart; - int nullByteEnd = arryByteStart + (arraySize+7) / 8; + int arryByteStart = start + vInt.length; + int nullByteCur = arryByteStart; + int nullByteEnd = arryByteStart + (arraySize + 7) / 8; // the begin the real elements int lastElementByteEnd = nullByteEnd; // the list element object inspector - ObjectInspector listEleObjectInspector = - ((ListObjectInspector)oi).getListElementObjectInspector(); + ObjectInspector listEleObjectInspector = ((ListObjectInspector) oi) + .getListElementObjectInspector(); // parsing elements one by one - for (int i=0; i= arraySize) { - return null; - } - return uncheckedGetElement(index); - } - - /** - * Get the element without checking out-of-bound index. - * @param index index to the array element - */ - private Object uncheckedGetElement(int index) { - - if (elementIsNull[index]) { - return null; - } else { - if (!elementInited[index]) { - elementInited[index] = true; - if (arrayElements[index] == null) { - arrayElements[index] = LazyBinaryFactory.createLazyBinaryObject( - ((LazyBinaryListObjectInspector)oi).getListElementObjectInspector()); - } - arrayElements[index].init(bytes, elementStart[index], elementLength[index]); - } - } - return arrayElements[index].getObject(); - } - - /** - * Returns the array size. - */ - public int getListLength() { - if (!parsed) { - parse(); - } - return arraySize; - } - - /** - * cachedList is reused every time getList is called. - * Different LazyBianryArray instances cannot share - * the same cachedList. - */ - ArrayList cachedList; - - /** Returns the List of actual primitive objects. - * Returns null for null array. - */ - public List getList() { - if (!parsed) { - parse(); - } - if (cachedList == null) { - cachedList = new ArrayList(arraySize); - } else { - cachedList.clear(); - } - for (int index=0; index= arraySize) { + return null; + } + return uncheckedGetElement(index); + } + + /** + * Get the element without checking out-of-bound index. + * + * @param index + * index to the array element + */ + private Object uncheckedGetElement(int index) { + + if (elementIsNull[index]) { + return null; + } else { + if (!elementInited[index]) { + elementInited[index] = true; + if (arrayElements[index] == null) { + arrayElements[index] = LazyBinaryFactory.createLazyBinaryObject((oi) + .getListElementObjectInspector()); + } + arrayElements[index].init(bytes, elementStart[index], + elementLength[index]); + } + } + return arrayElements[index].getObject(); + } + + /** + * Returns the array size. + */ + public int getListLength() { + if (!parsed) { + parse(); + } + return arraySize; + } + + /** + * cachedList is reused every time getList is called. Different + * LazyBianryArray instances cannot share the same cachedList. + */ + ArrayList cachedList; + + /** + * Returns the List of actual primitive objects. Returns null for null array. + */ + public List getList() { + if (!parsed) { + parse(); + } + if (cachedList == null) { + cachedList = new ArrayList(arraySize); + } else { + cachedList.clear(); + } + for (int index = 0; index < arraySize; index++) { + cachedList.add(uncheckedGetElement(index)); + } + return cachedList; + } } Index: serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryBoolean.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryBoolean.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryBoolean.java (working copy) @@ -34,7 +34,8 @@ *

* */ -public class LazyBinaryBoolean extends LazyBinaryPrimitive { +public class LazyBinaryBoolean extends + LazyBinaryPrimitive { public LazyBinaryBoolean(WritableBooleanObjectInspector oi) { super(oi); @@ -48,12 +49,12 @@ @Override public void init(ByteArrayRef bytes, int start, int length) { - assert(1 == length); + assert (1 == length); byte val = bytes.getData()[start]; if (val == 0) { - data.set(false); + data.set(false); } else if (val == 1) { - data.set(true); + data.set(true); } } } Index: serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryNonPrimitive.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryNonPrimitive.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryNonPrimitive.java (working copy) @@ -21,13 +21,14 @@ import org.apache.hadoop.hive.serde2.lazy.LazyUtils; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -public abstract class LazyBinaryNonPrimitive extends LazyBinaryObject { +public abstract class LazyBinaryNonPrimitive + extends LazyBinaryObject { protected ByteArrayRef bytes; protected int start; protected int length; - protected LazyBinaryNonPrimitive(OI oi) { + protected LazyBinaryNonPrimitive(OI oi) { super(oi); bytes = null; start = 0; @@ -51,7 +52,8 @@ this.start = start; this.length = length; } - + + @Override public int hashCode() { return LazyUtils.hashBytes(bytes.getData(), start, length); } Index: serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryStruct.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryStruct.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryStruct.java (working copy) @@ -30,18 +30,18 @@ import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; /** - * LazyBinaryStruct is serialized as follows: - * start A B A B A B end - * bytes[] -> |-----|---------|--- ... ---|-----|---------| + * LazyBinaryStruct is serialized as follows: start A B A B A B end bytes[] -> + * |-----|---------|--- ... ---|-----|---------| * - * Section A is one null-byte, corresponding to eight struct fields in Section B. - * Each bit indicates whether the corresponding field is null (0) or not null (1). - * Each field is a LazyBinaryObject. + * Section A is one null-byte, corresponding to eight struct fields in Section + * B. Each bit indicates whether the corresponding field is null (0) or not null + * (1). Each field is a LazyBinaryObject. * - * Following B, there is another section A and B. This pattern repeats until the + * Following B, there is another section A and B. This pattern repeats until the * all struct fields are serialized. */ -public class LazyBinaryStruct extends LazyBinaryNonPrimitive { +public class LazyBinaryStruct extends + LazyBinaryNonPrimitive { private static Log LOG = LogFactory.getLog(LazyBinaryStruct.class.getName()); @@ -54,26 +54,25 @@ * The fields of the struct. */ LazyBinaryObject[] fields; - + /** * Whether a field is initialized or not. */ boolean[] fieldInited; - + /** - * Whether a field is null or not. - * Because length is 0 does not means the field is null. - * In particular, a 0-length string is not null. + * Whether a field is null or not. Because length is 0 does not means the + * field is null. In particular, a 0-length string is not null. */ boolean[] fieldIsNull; - + /** - * The start positions and lengths of struct fields. - * Only valid when the data is parsed. + * The start positions and lengths of struct fields. Only valid when the data + * is parsed. */ - int[] fieldStart; + int[] fieldStart; int[] fieldLength; - + /** * Construct a LazyBinaryStruct object with an ObjectInspector. */ @@ -84,77 +83,81 @@ @Override public void init(ByteArrayRef bytes, int start, int length) { super.init(bytes, start, length); - parsed = false; + parsed = false; } - - RecordInfo recordInfo = new LazyBinaryUtils.RecordInfo(); + + RecordInfo recordInfo = new LazyBinaryUtils.RecordInfo(); boolean missingFieldWarned = false; boolean extraFieldWarned = false; + /** - * Parse the byte[] and fill fieldStart, fieldLength, - * fieldInited and fieldIsNull. + * Parse the byte[] and fill fieldStart, fieldLength, fieldInited and + * fieldIsNull. */ private void parse() { - - List fieldRefs = ((StructObjectInspector)oi).getAllStructFieldRefs(); - - if (fields == null) { + + List fieldRefs = ((StructObjectInspector) oi) + .getAllStructFieldRefs(); + + if (fields == null) { fields = new LazyBinaryObject[fieldRefs.size()]; - for (int i = 0 ; i < fields.length; i++) { - fields[i] = LazyBinaryFactory.createLazyBinaryObject(fieldRefs.get(i).getFieldObjectInspector()); + for (int i = 0; i < fields.length; i++) { + fields[i] = LazyBinaryFactory.createLazyBinaryObject(fieldRefs.get(i) + .getFieldObjectInspector()); } fieldInited = new boolean[fields.length]; fieldIsNull = new boolean[fields.length]; - fieldStart = new int[fields.length]; + fieldStart = new int[fields.length]; fieldLength = new int[fields.length]; } - + /** - * Please note that one null byte is followed by eight fields, - * then more null byte and fields. + * Please note that one null byte is followed by eight fields, then more + * null byte and fields. */ - + int fieldId = 0; - int structByteEnd = start + length; + int structByteEnd = start + length; byte[] bytes = this.bytes.getData(); - byte nullByte = bytes[start]; - int lastFieldByteEnd = start + 1; + byte nullByte = bytes[start]; + int lastFieldByteEnd = start + 1; // Go through all bytes in the byte[] - for (int i=0; i structByteEnd) { missingFieldWarned = true; @@ -162,21 +165,22 @@ + "only got " + fieldId + "! Ignoring similar problems."); } - Arrays.fill(fieldInited, false); - parsed = true; + Arrays.fill(fieldInited, false); + parsed = true; } - + /** * Get one field out of the struct. * - * If the field is a primitive field, return the actual object. - * Otherwise return the LazyObject. This is because PrimitiveObjectInspector - * does not have control over the object used by the user - the user simply - * directly use the Object instead of going through - * Object PrimitiveObjectInspector.get(Object). + * If the field is a primitive field, return the actual object. Otherwise + * return the LazyObject. This is because PrimitiveObjectInspector does not + * have control over the object used by the user - the user simply directly + * use the Object instead of going through Object + * PrimitiveObjectInspector.get(Object). * - * @param fieldID The field ID - * @return The field as a LazyObject + * @param fieldID + * The field ID + * @return The field as a LazyObject */ public Object getField(int fieldID) { if (!parsed) { @@ -184,15 +188,17 @@ } return uncheckedGetField(fieldID); } - + /** - * Get the field out of the row without checking parsed. - * This is called by both getField and getFieldsAsList. - * @param fieldID The id of the field starting from 0. - * @return The value of the field + * Get the field out of the row without checking parsed. This is called by + * both getField and getFieldsAsList. + * + * @param fieldID + * The id of the field starting from 0. + * @return The value of the field */ private Object uncheckedGetField(int fieldID) { - // Test the length first so in most cases we avoid doing a byte[] + // Test the length first so in most cases we avoid doing a byte[] // comparison. if (fieldIsNull[fieldID]) { return null; @@ -205,8 +211,10 @@ } ArrayList cachedList; + /** * Get the values of the fields as an ArrayList. + * * @return The values of the fields as an ArrayList. */ public ArrayList getFieldsAsList() { @@ -218,12 +226,12 @@ } else { cachedList.clear(); } - for (int i=0; i", - "\" \"", - "\"\\t\"", - "\"\\n\"", - "\"\\r\"", - "", - "", - "", - "\"const\"", - "\"namespace\"", - "\"cpp_namespace\"", - "\"cpp_include\"", - "\"cpp_type\"", - "\"java_package\"", - "\"cocoa_prefix\"", - "\"csharp_namespace\"", - "\"php_namespace\"", - "\"py_module\"", - "\"perl_package\"", - "\"ruby_namespace\"", - "\"smalltalk_category\"", - "\"smalltalk_prefix\"", - "\"xsd_all\"", - "\"xsd_optional\"", - "\"xsd_nillable\"", - "\"xsd_namespace\"", - "\"xsd_attrs\"", - "\"include\"", - "\"void\"", - "\"bool\"", - "\"byte\"", - "\"i16\"", - "\"i32\"", - "\"i64\"", - "\"double\"", - "\"string\"", - "\"slist\"", - "\"senum\"", - "\"map\"", - "\"list\"", - "\"set\"", - "\"async\"", - "\"typedef\"", - "\"struct\"", - "\"exception\"", - "\"extends\"", - "\"throws\"", - "\"service\"", - "\"enum\"", - "\"required\"", - "\"optional\"", - "\"skip\"", - "", - "", - "", - "", - "", - "", - "", - "\",\"", - "\";\"", - "\"{\"", - "\"}\"", - "\"=\"", - "\"[\"", - "\"]\"", - "\":\"", - "\"(\"", - "\")\"", - "\"<\"", - "\">\"", - }; + String[] tokenImage = { "", "\" \"", "\"\\t\"", "\"\\n\"", "\"\\r\"", + "", "", "", + "\"const\"", "\"namespace\"", "\"cpp_namespace\"", "\"cpp_include\"", + "\"cpp_type\"", "\"java_package\"", "\"cocoa_prefix\"", + "\"csharp_namespace\"", "\"php_namespace\"", "\"py_module\"", + "\"perl_package\"", "\"ruby_namespace\"", "\"smalltalk_category\"", + "\"smalltalk_prefix\"", "\"xsd_all\"", "\"xsd_optional\"", + "\"xsd_nillable\"", "\"xsd_namespace\"", "\"xsd_attrs\"", "\"include\"", + "\"void\"", "\"bool\"", "\"byte\"", "\"i16\"", "\"i32\"", "\"i64\"", + "\"double\"", "\"string\"", "\"slist\"", "\"senum\"", "\"map\"", + "\"list\"", "\"set\"", "\"async\"", "\"typedef\"", "\"struct\"", + "\"exception\"", "\"extends\"", "\"throws\"", "\"service\"", "\"enum\"", + "\"required\"", "\"optional\"", "\"skip\"", "", + "", "", "", "", + "", "", "\",\"", "\";\"", "\"{\"", + "\"}\"", "\"=\"", "\"[\"", "\"]\"", "\":\"", "\"(\"", "\")\"", "\"<\"", + "\">\"", }; } Index: serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDe.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDe.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDe.java (working copy) @@ -18,28 +18,33 @@ package org.apache.hadoop.hive.serde2.dynamic_type; +import java.io.ByteArrayInputStream; +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; +import java.util.Properties; + import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.serde.Constants; -import org.apache.hadoop.hive.serde2.*; +import org.apache.hadoop.hive.serde2.ByteStream; +import org.apache.hadoop.hive.serde2.SerDe; +import org.apache.hadoop.hive.serde2.SerDeException; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; - import org.apache.hadoop.hive.serde2.thrift.ConfigurableTProtocol; import org.apache.hadoop.hive.serde2.thrift.TReflectionUtils; - -import java.util.*; -import java.io.*; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.io.*; +import org.apache.hadoop.io.BytesWritable; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.io.Writable; import org.apache.hadoop.util.StringUtils; +import org.apache.thrift.protocol.TProtocol; +import org.apache.thrift.protocol.TProtocolFactory; +import org.apache.thrift.transport.TIOStreamTransport; -import org.apache.thrift.protocol.*; -import org.apache.thrift.transport.*; - public class DynamicSerDe implements SerDe, Serializable { public static final Log LOG = LogFactory.getLog(DynamicSerDe.class.getName()); @@ -54,30 +59,34 @@ transient protected ByteStream.Output bos_; /** - * protocols are protected in case any of their properties need to be queried from another - * class in this package. For TCTLSeparatedProtocol for example, may want to query the separators. + * protocols are protected in case any of their properties need to be queried + * from another class in this package. For TCTLSeparatedProtocol for example, + * may want to query the separators. */ transient protected TProtocol oprot_; transient protected TProtocol iprot_; TIOStreamTransport tios; - public void initialize(Configuration job, Properties tbl) throws SerDeException { + public void initialize(Configuration job, Properties tbl) + throws SerDeException { try { String ddl = tbl.getProperty(Constants.SERIALIZATION_DDL); type_name = tbl.getProperty(META_TABLE_NAME); String protoName = tbl.getProperty(Constants.SERIALIZATION_FORMAT); - if(protoName == null) { + if (protoName == null) { protoName = "org.apache.thrift.protocol.TBinaryProtocol"; } // For backward compatibility - protoName = protoName.replace("com.facebook.thrift.protocol", "org.apache.thrift.protocol"); - TProtocolFactory protFactory = TReflectionUtils.getProtocolFactoryByName(protoName); + protoName = protoName.replace("com.facebook.thrift.protocol", + "org.apache.thrift.protocol"); + TProtocolFactory protFactory = TReflectionUtils + .getProtocolFactoryByName(protoName); bos_ = new ByteStream.Output(); bis_ = new ByteStream.Input(); - tios = new TIOStreamTransport(bis_,bos_); + tios = new TIOStreamTransport(bis_, bos_); oprot_ = protFactory.getProtocol(tios); iprot_ = protFactory.getProtocol(tios); @@ -86,32 +95,34 @@ * initialize the protocols */ - if(oprot_ instanceof org.apache.hadoop.hive.serde2.thrift.ConfigurableTProtocol) { - ((ConfigurableTProtocol)oprot_).initialize(job, tbl); + if (oprot_ instanceof org.apache.hadoop.hive.serde2.thrift.ConfigurableTProtocol) { + ((ConfigurableTProtocol) oprot_).initialize(job, tbl); } - if(iprot_ instanceof org.apache.hadoop.hive.serde2.thrift.ConfigurableTProtocol) { - ((ConfigurableTProtocol)iprot_).initialize(job, tbl); + if (iprot_ instanceof org.apache.hadoop.hive.serde2.thrift.ConfigurableTProtocol) { + ((ConfigurableTProtocol) iprot_).initialize(job, tbl); } // in theory the include path should come from the configuration List include_path = new ArrayList(); include_path.add("."); LOG.debug("ddl=" + ddl); - this.parse_tree = new thrift_grammar(new ByteArrayInputStream(ddl.getBytes()), include_path,false); - this.parse_tree.Start(); + parse_tree = new thrift_grammar(new ByteArrayInputStream(ddl.getBytes()), + include_path, false); + parse_tree.Start(); - this.bt = (DynamicSerDeStructBase)this.parse_tree.types.get(type_name); + bt = (DynamicSerDeStructBase) parse_tree.types.get(type_name); - if(this.bt == null) { - this.bt = (DynamicSerDeStructBase)this.parse_tree.tables.get(type_name); + if (bt == null) { + bt = (DynamicSerDeStructBase) parse_tree.tables.get(type_name); } - if(this.bt == null) { - throw new SerDeException("Could not lookup table type " + type_name + " in this ddl: " + ddl); + if (bt == null) { + throw new SerDeException("Could not lookup table type " + type_name + + " in this ddl: " + ddl); } - this.bt.initialize(); + bt.initialize(); } catch (Exception e) { System.err.println(StringUtils.stringifyException(e)); throw new SerDeException(e); @@ -119,53 +130,59 @@ } Object deserializeReuse = null; + public Object deserialize(Writable field) throws SerDeException { try { if (field instanceof Text) { - Text b = (Text)field; + Text b = (Text) field; bis_.reset(b.getBytes(), b.getLength()); } else { - BytesWritable b = (BytesWritable)field; + BytesWritable b = (BytesWritable) field; bis_.reset(b.get(), b.getSize()); } - deserializeReuse = this.bt.deserialize(deserializeReuse, iprot_); + deserializeReuse = bt.deserialize(deserializeReuse, iprot_); return deserializeReuse; - } catch(Exception e) { + } catch (Exception e) { e.printStackTrace(); throw new SerDeException(e); } } - public static ObjectInspector dynamicSerDeStructBaseToObjectInspector(DynamicSerDeTypeBase bt) throws SerDeException { + public static ObjectInspector dynamicSerDeStructBaseToObjectInspector( + DynamicSerDeTypeBase bt) throws SerDeException { if (bt.isList()) { - return ObjectInspectorFactory.getStandardListObjectInspector( - dynamicSerDeStructBaseToObjectInspector(((DynamicSerDeTypeList)bt).getElementType())); + return ObjectInspectorFactory + .getStandardListObjectInspector(dynamicSerDeStructBaseToObjectInspector(((DynamicSerDeTypeList) bt) + .getElementType())); } else if (bt.isMap()) { - DynamicSerDeTypeMap btMap = (DynamicSerDeTypeMap)bt; + DynamicSerDeTypeMap btMap = (DynamicSerDeTypeMap) bt; return ObjectInspectorFactory.getStandardMapObjectInspector( - dynamicSerDeStructBaseToObjectInspector(btMap.getKeyType()), + dynamicSerDeStructBaseToObjectInspector(btMap.getKeyType()), dynamicSerDeStructBaseToObjectInspector(btMap.getValueType())); } else if (bt.isPrimitive()) { - return PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector( - PrimitiveObjectInspectorUtils.getTypeEntryFromPrimitiveJavaClass(bt.getRealType()).primitiveCategory); + return PrimitiveObjectInspectorFactory + .getPrimitiveJavaObjectInspector(PrimitiveObjectInspectorUtils + .getTypeEntryFromPrimitiveJavaClass(bt.getRealType()).primitiveCategory); } else { // Must be a struct - DynamicSerDeStructBase btStruct = (DynamicSerDeStructBase)bt; + DynamicSerDeStructBase btStruct = (DynamicSerDeStructBase) bt; DynamicSerDeFieldList fieldList = btStruct.getFieldList(); DynamicSerDeField[] fields = fieldList.getChildren(); - ArrayList fieldNames = new ArrayList(fields.length); - ArrayList fieldObjectInspectors = new ArrayList(fields.length); - for(int i=0; i fieldNames = new ArrayList(fields.length); + ArrayList fieldObjectInspectors = new ArrayList( + fields.length); + for (DynamicSerDeField field : fields) { + fieldNames.add(field.name); + fieldObjectInspectors.add(dynamicSerDeStructBaseToObjectInspector(field + .getFieldType().getMyType())); } - return ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldObjectInspectors); + return ObjectInspectorFactory.getStandardStructObjectInspector( + fieldNames, fieldObjectInspectors); } } public ObjectInspector getObjectInspector() throws SerDeException { - return dynamicSerDeStructBaseToObjectInspector(this.bt); + return dynamicSerDeStructBaseToObjectInspector(bt); } public Class getSerializedClass() { @@ -173,17 +190,18 @@ } BytesWritable ret = new BytesWritable(); + public Writable serialize(Object obj, ObjectInspector objInspector) - throws SerDeException { + throws SerDeException { try { bos_.reset(); - this.bt.serialize(obj, objInspector, oprot_); + bt.serialize(obj, objInspector, oprot_); oprot_.getTransport().flush(); - } catch(Exception e) { + } catch (Exception e) { e.printStackTrace(); throw new SerDeException(e); } - ret.set(bos_.getData(),0,bos_.getCount()); + ret.set(bos_.getData(), 0, bos_.getCount()); return ret; } } Index: serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeField.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeField.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeField.java (working copy) @@ -18,34 +18,31 @@ package org.apache.hadoop.hive.serde2.dynamic_type; -import org.apache.hadoop.hive.serde2.*; -import org.apache.thrift.TException; -import org.apache.thrift.protocol.TProtocol; public class DynamicSerDeField extends DynamicSerDeSimpleNode { - // production is: - // [this.fieldid :] Requiredness() FieldType() this.name FieldValue() [CommaOrSemicolon()] + // [this.fieldid :] Requiredness() FieldType() this.name FieldValue() + // [CommaOrSemicolon()] private final int FD_REQUIREDNESS = 0; private final int FD_FIELD_TYPE = 1; - private final int FD_FIELD_VALUE =2; public boolean isSkippable() { - return ((DynamicSerDeFieldRequiredness)this.jjtGetChild(FD_REQUIREDNESS)).getRequiredness() == DynamicSerDeFieldRequiredness.RequirednessTypes.Skippable; + return ((DynamicSerDeFieldRequiredness) jjtGetChild(FD_REQUIREDNESS)) + .getRequiredness() == DynamicSerDeFieldRequiredness.RequirednessTypes.Skippable; } public DynamicSerDeFieldType getFieldType() { - return (DynamicSerDeFieldType)this.jjtGetChild(FD_FIELD_TYPE); + return (DynamicSerDeFieldType) jjtGetChild(FD_FIELD_TYPE); } public DynamicSerDeField(int i) { super(i); } + public DynamicSerDeField(thrift_grammar p, int i) { - super(p,i); + super(p, i); } } - Index: serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypeBool.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypeBool.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypeBool.java (working copy) @@ -18,13 +18,11 @@ package org.apache.hadoop.hive.serde2.dynamic_type; -import org.apache.thrift.TException; -import org.apache.thrift.protocol.*; -import org.apache.hadoop.hive.serde2.*; +import org.apache.hadoop.hive.serde2.SerDeException; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.BooleanObjectInspector; - +import org.apache.thrift.TException; +import org.apache.thrift.protocol.TProtocol; import org.apache.thrift.protocol.TType; public class DynamicSerDeTypeBool extends DynamicSerDeTypeBase { @@ -39,6 +37,7 @@ super(p, i); } + @Override public String toString() { return "bool"; } @@ -47,8 +46,10 @@ public Object deserialize(Object reuse, TProtocol iprot) throws SerDeException, TException, IllegalAccessException { boolean val = iprot.readBool(); - if (val == false && iprot instanceof org.apache.hadoop.hive.serde2.thrift.WriteNullsProtocol && - ((org.apache.hadoop.hive.serde2.thrift.WriteNullsProtocol)iprot).lastPrimitiveWasNull()) { + if (val == false + && iprot instanceof org.apache.hadoop.hive.serde2.thrift.WriteNullsProtocol + && ((org.apache.hadoop.hive.serde2.thrift.WriteNullsProtocol) iprot) + .lastPrimitiveWasNull()) { return null; } return Boolean.valueOf(val); @@ -62,11 +63,18 @@ oprot.writeBool(poi.get(o)); } + @Override public byte getType() { return TType.BOOL; } - - public Class getRealType() { return java.lang.Boolean.class; } - public Boolean getRealTypeInstance() { return Boolean.FALSE; } - + + @Override + public Class getRealType() { + return java.lang.Boolean.class; + } + + public Boolean getRealTypeInstance() { + return Boolean.FALSE; + } + } Index: serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypei32.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypei32.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypei32.java (working copy) @@ -18,20 +18,11 @@ package org.apache.hadoop.hive.serde2.dynamic_type; -import org.apache.thrift.TException; -import org.apache.thrift.TApplicationException; -import org.apache.thrift.protocol.*; -import org.apache.thrift.server.*; -import org.apache.thrift.transport.*; -import java.util.*; -import java.io.*; -import org.apache.hadoop.hive.serde2.*; +import org.apache.hadoop.hive.serde2.SerDeException; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.IntObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.ShortObjectInspector; - -import java.lang.reflect.*; +import org.apache.thrift.TException; +import org.apache.thrift.protocol.TProtocol; import org.apache.thrift.protocol.TType; public class DynamicSerDeTypei32 extends DynamicSerDeTypeBase { @@ -41,32 +32,47 @@ public DynamicSerDeTypei32(int i) { super(i); } + public DynamicSerDeTypei32(thrift_grammar p, int i) { - super(p,i); + super(p, i); } - public String toString() { return "i32"; } + @Override + public String toString() { + return "i32"; + } @Override - public Object deserialize(Object reuse, TProtocol iprot) throws SerDeException, TException, IllegalAccessException { + public Object deserialize(Object reuse, TProtocol iprot) + throws SerDeException, TException, IllegalAccessException { int val = iprot.readI32(); - if (val == 0 && iprot instanceof org.apache.hadoop.hive.serde2.thrift.WriteNullsProtocol && - ((org.apache.hadoop.hive.serde2.thrift.WriteNullsProtocol)iprot).lastPrimitiveWasNull()) { + if (val == 0 + && iprot instanceof org.apache.hadoop.hive.serde2.thrift.WriteNullsProtocol + && ((org.apache.hadoop.hive.serde2.thrift.WriteNullsProtocol) iprot) + .lastPrimitiveWasNull()) { return null; } return Integer.valueOf(val); } @Override - public void serialize(Object o, ObjectInspector oi, TProtocol oprot) throws TException, SerDeException, NoSuchFieldException,IllegalAccessException { + public void serialize(Object o, ObjectInspector oi, TProtocol oprot) + throws TException, SerDeException, NoSuchFieldException, + IllegalAccessException { IntObjectInspector poi = (IntObjectInspector) oi; oprot.writeI32(poi.get(o)); } - public Class getRealType() { return java.lang.Integer.class; } - public Integer getRealTypeInstance() { return Integer.valueOf(0); } + @Override + public Class getRealType() { + return java.lang.Integer.class; + } + public Integer getRealTypeInstance() { + return Integer.valueOf(0); + } + @Override public byte getType() { return TType.I32; } Index: serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypedef.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypedef.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypedef.java (working copy) @@ -18,19 +18,11 @@ package org.apache.hadoop.hive.serde2.dynamic_type; -import org.apache.thrift.TException; -import org.apache.thrift.TApplicationException; -import org.apache.thrift.protocol.*; -import org.apache.thrift.server.*; -import org.apache.thrift.transport.*; -import java.util.*; -import java.io.*; -import org.apache.hadoop.hive.serde2.*; +import org.apache.hadoop.hive.serde2.SerDeException; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.thrift.TException; +import org.apache.thrift.protocol.TProtocol; -import java.lang.reflect.*; -import org.apache.thrift.protocol.TType.*; - public class DynamicSerDeTypedef extends DynamicSerDeTypeBase { // production is: typedef DefinitionType() this.name @@ -40,41 +32,44 @@ public DynamicSerDeTypedef(int i) { super(i); } + public DynamicSerDeTypedef(thrift_grammar p, int i) { - super(p,i); + super(p, i); } private DynamicSerDeSimpleNode getDefinitionType() { - return (DynamicSerDeSimpleNode)this.jjtGetChild(FD_DEFINITION_TYPE); + return (DynamicSerDeSimpleNode) jjtGetChild(FD_DEFINITION_TYPE); } - public DynamicSerDeTypeBase getMyType() { - DynamicSerDeSimpleNode child = this.getDefinitionType(); - DynamicSerDeTypeBase ret = (DynamicSerDeTypeBase)child.jjtGetChild(0); + DynamicSerDeSimpleNode child = getDefinitionType(); + DynamicSerDeTypeBase ret = (DynamicSerDeTypeBase) child.jjtGetChild(0); return ret; } + @Override public String toString() { - String result = "typedef " + this.name + "("; - result += this.getDefinitionType().toString(); + String result = "typedef " + name + "("; + result += getDefinitionType().toString(); result += ")"; return result; } + @Override public byte getType() { throw new RuntimeException("not implemented"); } @Override public Object deserialize(Object reuse, TProtocol iprot) - throws SerDeException, TException, IllegalAccessException { + throws SerDeException, TException, IllegalAccessException { throw new RuntimeException("not implemented"); } + @Override public void serialize(Object o, ObjectInspector oi, TProtocol oprot) - throws TException, SerDeException, NoSuchFieldException, - IllegalAccessException { + throws TException, SerDeException, NoSuchFieldException, + IllegalAccessException { throw new RuntimeException("not implemented"); } Index: serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypei16.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypei16.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypei16.java (working copy) @@ -18,53 +18,57 @@ package org.apache.hadoop.hive.serde2.dynamic_type; -import org.apache.thrift.TException; -import org.apache.thrift.TApplicationException; -import org.apache.thrift.protocol.*; -import org.apache.thrift.server.*; -import org.apache.thrift.transport.*; -import java.util.*; -import java.io.*; -import org.apache.hadoop.hive.serde2.*; +import org.apache.hadoop.hive.serde2.SerDeException; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.DoubleObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.ShortObjectInspector; - -import java.lang.reflect.*; +import org.apache.thrift.TException; +import org.apache.thrift.protocol.TProtocol; import org.apache.thrift.protocol.TType; public class DynamicSerDeTypei16 extends DynamicSerDeTypeBase { - public Class getRealType() { return Integer.valueOf(2).getClass(); } + @Override + public Class getRealType() { + return Integer.valueOf(2).getClass(); + } // production is: i16 public DynamicSerDeTypei16(int i) { super(i); } + public DynamicSerDeTypei16(thrift_grammar p, int i) { - super(p,i); + super(p, i); } - public String toString() { return "i16"; } + @Override + public String toString() { + return "i16"; + } @Override - public Object deserialize(Object reuse, TProtocol iprot) throws SerDeException, TException, IllegalAccessException { + public Object deserialize(Object reuse, TProtocol iprot) + throws SerDeException, TException, IllegalAccessException { int val = iprot.readI16(); - if (val == 0 && iprot instanceof org.apache.hadoop.hive.serde2.thrift.WriteNullsProtocol && - ((org.apache.hadoop.hive.serde2.thrift.WriteNullsProtocol)iprot).lastPrimitiveWasNull()) { + if (val == 0 + && iprot instanceof org.apache.hadoop.hive.serde2.thrift.WriteNullsProtocol + && ((org.apache.hadoop.hive.serde2.thrift.WriteNullsProtocol) iprot) + .lastPrimitiveWasNull()) { return null; } return Integer.valueOf(val); } @Override - public void serialize(Object o, ObjectInspector oi, TProtocol oprot) throws TException, SerDeException, NoSuchFieldException,IllegalAccessException { + public void serialize(Object o, ObjectInspector oi, TProtocol oprot) + throws TException, SerDeException, NoSuchFieldException, + IllegalAccessException { ShortObjectInspector poi = (ShortObjectInspector) oi; oprot.writeI16(poi.get(o)); } + @Override public byte getType() { return TType.I16; } Index: serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/ParseException.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/ParseException.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/ParseException.java (working copy) @@ -2,33 +2,27 @@ package org.apache.hadoop.hive.serde2.dynamic_type; /** - * This exception is thrown when parse errors are encountered. - * You can explicitly create objects of this exception type by - * calling the method generateParseException in the generated - * parser. - * - * You can modify this class to customize your error reporting - * mechanisms so long as you retain the public fields. + * This exception is thrown when parse errors are encountered. You can + * explicitly create objects of this exception type by calling the method + * generateParseException in the generated parser. + * + * You can modify this class to customize your error reporting mechanisms so + * long as you retain the public fields. */ public class ParseException extends Exception { /** - * This constructor is used by the method "generateParseException" - * in the generated parser. Calling this constructor generates - * a new object of this type with the fields "currentToken", - * "expectedTokenSequences", and "tokenImage" set. The boolean - * flag "specialConstructor" is also set to true to indicate that - * this constructor was used to create this object. - * This constructor calls its super class with the empty string - * to force the "toString" method of parent class "Throwable" to - * print the error message in the form: - * ParseException: + * This constructor is used by the method "generateParseException" in the + * generated parser. Calling this constructor generates a new object of this + * type with the fields "currentToken", "expectedTokenSequences", and + * "tokenImage" set. The boolean flag "specialConstructor" is also set to true + * to indicate that this constructor was used to create this object. This + * constructor calls its super class with the empty string to force the + * "toString" method of parent class "Throwable" to print the error message in + * the form: ParseException: */ public ParseException(Token currentTokenVal, - int[][] expectedTokenSequencesVal, - String[] tokenImageVal - ) - { + int[][] expectedTokenSequencesVal, String[] tokenImageVal) { super(""); specialConstructor = true; currentToken = currentTokenVal; @@ -37,13 +31,12 @@ } /** - * The following constructors are for use by you for whatever - * purpose you can think of. Constructing the exception in this - * manner makes the exception behave in the normal way - i.e., as - * documented in the class "Throwable". The fields "errorToken", - * "expectedTokenSequences", and "tokenImage" do not contain - * relevant information. The JavaCC generated code does not use - * these constructors. + * The following constructors are for use by you for whatever purpose you can + * think of. Constructing the exception in this manner makes the exception + * behave in the normal way - i.e., as documented in the class "Throwable". + * The fields "errorToken", "expectedTokenSequences", and "tokenImage" do not + * contain relevant information. The JavaCC generated code does not use these + * constructors. */ public ParseException() { @@ -57,57 +50,56 @@ } /** - * This variable determines which constructor was used to create - * this object and thereby affects the semantics of the - * "getMessage" method (see below). + * This variable determines which constructor was used to create this object + * and thereby affects the semantics of the "getMessage" method (see below). */ protected boolean specialConstructor; /** - * This is the last token that has been consumed successfully. If - * this object has been created due to a parse error, the token - * followng this token will (therefore) be the first error token. + * This is the last token that has been consumed successfully. If this object + * has been created due to a parse error, the token followng this token will + * (therefore) be the first error token. */ public Token currentToken; /** - * Each entry in this array is an array of integers. Each array - * of integers represents a sequence of tokens (by their ordinal - * values) that is expected at this point of the parse. + * Each entry in this array is an array of integers. Each array of integers + * represents a sequence of tokens (by their ordinal values) that is expected + * at this point of the parse. */ public int[][] expectedTokenSequences; /** - * This is a reference to the "tokenImage" array of the generated - * parser within which the parse error occurred. This array is - * defined in the generated ...Constants interface. + * This is a reference to the "tokenImage" array of the generated parser + * within which the parse error occurred. This array is defined in the + * generated ...Constants interface. */ public String[] tokenImage; /** - * This method has the standard behavior when this object has been - * created using the standard constructors. Otherwise, it uses - * "currentToken" and "expectedTokenSequences" to generate a parse - * error message and returns it. If this object has been created - * due to a parse error, and you do not catch it (it gets thrown - * from the parser), then this method is called during the printing - * of the final stack trace, and hence the correct error message - * gets displayed. + * This method has the standard behavior when this object has been created + * using the standard constructors. Otherwise, it uses "currentToken" and + * "expectedTokenSequences" to generate a parse error message and returns it. + * If this object has been created due to a parse error, and you do not catch + * it (it gets thrown from the parser), then this method is called during the + * printing of the final stack trace, and hence the correct error message gets + * displayed. */ + @Override public String getMessage() { if (!specialConstructor) { return super.getMessage(); } StringBuffer expected = new StringBuffer(); int maxSize = 0; - for (int i = 0; i < expectedTokenSequences.length; i++) { - if (maxSize < expectedTokenSequences[i].length) { - maxSize = expectedTokenSequences[i].length; + for (int[] expectedTokenSequence : expectedTokenSequences) { + if (maxSize < expectedTokenSequence.length) { + maxSize = expectedTokenSequence.length; } - for (int j = 0; j < expectedTokenSequences[i].length; j++) { - expected.append(tokenImage[expectedTokenSequences[i][j]]).append(" "); + for (int j = 0; j < expectedTokenSequence.length; j++) { + expected.append(tokenImage[expectedTokenSequence[j]]).append(" "); } - if (expectedTokenSequences[i][expectedTokenSequences[i].length - 1] != 0) { + if (expectedTokenSequence[expectedTokenSequence.length - 1] != 0) { expected.append("..."); } expected.append(eol).append(" "); @@ -115,15 +107,18 @@ String retval = "Encountered \""; Token tok = currentToken.next; for (int i = 0; i < maxSize; i++) { - if (i != 0) retval += " "; + if (i != 0) { + retval += " "; + } if (tok.kind == 0) { retval += tokenImage[0]; break; } retval += add_escapes(tok.image); - tok = tok.next; + tok = tok.next; } - retval += "\" at line " + currentToken.next.beginLine + ", column " + currentToken.next.beginColumn; + retval += "\" at line " + currentToken.next.beginLine + ", column " + + currentToken.next.beginColumn; retval += "." + eol; if (expectedTokenSequences.length == 1) { retval += "Was expecting:" + eol + " "; @@ -138,55 +133,53 @@ * The end of line string for this machine. */ protected String eol = System.getProperty("line.separator", "\n"); - + /** - * Used to convert raw characters to their escaped version - * when these raw version cannot be used as part of an ASCII - * string literal. + * Used to convert raw characters to their escaped version when these raw + * version cannot be used as part of an ASCII string literal. */ protected String add_escapes(String str) { - StringBuffer retval = new StringBuffer(); - char ch; - for (int i = 0; i < str.length(); i++) { - switch (str.charAt(i)) - { - case 0 : - continue; - case '\b': - retval.append("\\b"); - continue; - case '\t': - retval.append("\\t"); - continue; - case '\n': - retval.append("\\n"); - continue; - case '\f': - retval.append("\\f"); - continue; - case '\r': - retval.append("\\r"); - continue; - case '\"': - retval.append("\\\""); - continue; - case '\'': - retval.append("\\\'"); - continue; - case '\\': - retval.append("\\\\"); - continue; - default: - if ((ch = str.charAt(i)) < 0x20 || ch > 0x7e) { - String s = "0000" + Integer.toString(ch, 16); - retval.append("\\u" + s.substring(s.length() - 4, s.length())); - } else { - retval.append(ch); - } - continue; + StringBuffer retval = new StringBuffer(); + char ch; + for (int i = 0; i < str.length(); i++) { + switch (str.charAt(i)) { + case 0: + continue; + case '\b': + retval.append("\\b"); + continue; + case '\t': + retval.append("\\t"); + continue; + case '\n': + retval.append("\\n"); + continue; + case '\f': + retval.append("\\f"); + continue; + case '\r': + retval.append("\\r"); + continue; + case '\"': + retval.append("\\\""); + continue; + case '\'': + retval.append("\\\'"); + continue; + case '\\': + retval.append("\\\\"); + continue; + default: + if ((ch = str.charAt(i)) < 0x20 || ch > 0x7e) { + String s = "0000" + Integer.toString(ch, 16); + retval.append("\\u" + s.substring(s.length() - 4, s.length())); + } else { + retval.append(ch); } + continue; } - return retval.toString(); - } + } + return retval.toString(); + } } Index: serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypei64.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypei64.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypei64.java (working copy) @@ -18,53 +18,57 @@ package org.apache.hadoop.hive.serde2.dynamic_type; -import org.apache.thrift.TException; -import org.apache.thrift.TApplicationException; -import org.apache.thrift.protocol.*; -import org.apache.thrift.server.*; -import org.apache.thrift.transport.*; -import java.util.*; -import java.io.*; -import org.apache.hadoop.hive.serde2.*; +import org.apache.hadoop.hive.serde2.SerDeException; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.IntObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.LongObjectInspector; - -import java.lang.reflect.*; +import org.apache.thrift.TException; +import org.apache.thrift.protocol.TProtocol; import org.apache.thrift.protocol.TType; public class DynamicSerDeTypei64 extends DynamicSerDeTypeBase { - public Class getRealType() { return Long.valueOf(0).getClass(); } + @Override + public Class getRealType() { + return Long.valueOf(0).getClass(); + } // production is: i64 public DynamicSerDeTypei64(int i) { super(i); } + public DynamicSerDeTypei64(thrift_grammar p, int i) { - super(p,i); + super(p, i); } - public String toString() { return "i64"; } + @Override + public String toString() { + return "i64"; + } @Override - public Object deserialize(Object reuse, TProtocol iprot) throws SerDeException, TException, IllegalAccessException { + public Object deserialize(Object reuse, TProtocol iprot) + throws SerDeException, TException, IllegalAccessException { long val = iprot.readI64(); - if (val == 0 && iprot instanceof org.apache.hadoop.hive.serde2.thrift.WriteNullsProtocol && - ((org.apache.hadoop.hive.serde2.thrift.WriteNullsProtocol)iprot).lastPrimitiveWasNull()) { + if (val == 0 + && iprot instanceof org.apache.hadoop.hive.serde2.thrift.WriteNullsProtocol + && ((org.apache.hadoop.hive.serde2.thrift.WriteNullsProtocol) iprot) + .lastPrimitiveWasNull()) { return null; } return Long.valueOf(val); } @Override - public void serialize(Object o, ObjectInspector oi, TProtocol oprot) throws TException, SerDeException, NoSuchFieldException,IllegalAccessException { + public void serialize(Object o, ObjectInspector oi, TProtocol oprot) + throws TException, SerDeException, NoSuchFieldException, + IllegalAccessException { LongObjectInspector poi = (LongObjectInspector) oi; oprot.writeI64(poi.get(o)); } + @Override public byte getType() { return TType.I64; } Index: serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypeString.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypeString.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypeString.java (working copy) @@ -18,23 +18,14 @@ package org.apache.hadoop.hive.serde2.dynamic_type; -import org.apache.thrift.TException; -import org.apache.thrift.TApplicationException; -import org.apache.thrift.protocol.*; -import org.apache.thrift.server.*; -import org.apache.thrift.transport.*; -import java.util.*; -import java.io.*; -import org.apache.hadoop.hive.serde2.*; +import org.apache.hadoop.hive.serde2.SerDeException; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.StringObjectInspector; import org.apache.hadoop.hive.serde2.thrift.WriteTextProtocol; -import org.apache.hadoop.io.Text; +import org.apache.thrift.TException; +import org.apache.thrift.protocol.TProtocol; +import org.apache.thrift.protocol.TType; -import java.lang.reflect.*; -import org.apache.thrift.protocol.TType.*; - public class DynamicSerDeTypeString extends DynamicSerDeTypeBase { // production is: string @@ -42,32 +33,45 @@ public DynamicSerDeTypeString(int i) { super(i); } + public DynamicSerDeTypeString(thrift_grammar p, int i) { - super(p,i); + super(p, i); } - public Class getRealType() { return java.lang.String.class; } - public String toString() { return "string"; } + @Override + public Class getRealType() { + return java.lang.String.class; + } - public String deserialize(TProtocol iprot) throws SerDeException, TException, IllegalAccessException { + @Override + public String toString() { + return "string"; + } + + public String deserialize(TProtocol iprot) throws SerDeException, TException, + IllegalAccessException { return iprot.readString(); } @Override - public Object deserialize(Object reuse, TProtocol iprot) throws SerDeException, TException, IllegalAccessException { + public Object deserialize(Object reuse, TProtocol iprot) + throws SerDeException, TException, IllegalAccessException { return iprot.readString(); } - + @Override - public void serialize(Object o, ObjectInspector oi, TProtocol oprot) throws TException, SerDeException, NoSuchFieldException,IllegalAccessException { + public void serialize(Object o, ObjectInspector oi, TProtocol oprot) + throws TException, SerDeException, NoSuchFieldException, + IllegalAccessException { StringObjectInspector poi = (StringObjectInspector) oi; if (oprot instanceof WriteTextProtocol) { - ((WriteTextProtocol)oprot).writeText((Text)poi.getPrimitiveWritableObject(o)); + ((WriteTextProtocol) oprot).writeText(poi.getPrimitiveWritableObject(o)); } else { - oprot.writeString((String)poi.getPrimitiveJavaObject(o)); + oprot.writeString(poi.getPrimitiveJavaObject(o)); } } + @Override public byte getType() { return TType.STRING; } Index: serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/thrift_grammarTokenManager.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/thrift_grammarTokenManager.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/thrift_grammarTokenManager.java (working copy) @@ -1,1499 +1,1604 @@ /* Generated By:JJTree&JavaCC: Do not edit this line. thrift_grammarTokenManager.java */ package org.apache.hadoop.hive.serde2.dynamic_type; -import java.util.*; -import java.io.*; -import java.net.*; -import org.apache.thrift.protocol.*; -import org.apache.thrift.transport.*; -import org.apache.hadoop.hive.serde2.dynamic_type.*; /** Token Manager. */ -public class thrift_grammarTokenManager implements thrift_grammarConstants -{ +public class thrift_grammarTokenManager implements thrift_grammarConstants { /** Debug output. */ - public java.io.PrintStream debugStream = System.out; + public java.io.PrintStream debugStream = System.out; + /** Set debug output. */ - public void setDebugStream(java.io.PrintStream ds) { debugStream = ds; } -private final int jjStopStringLiteralDfa_0(int pos, long active0, long active1) -{ - switch (pos) - { - case 0: - if ((active0 & 0xfffffffffff00L) != 0L) - { - jjmatchedKind = 54; - return 35; - } - return -1; - case 1: - if ((active0 & 0xfffffffffff00L) != 0L) - { - jjmatchedKind = 54; - jjmatchedPos = 1; - return 35; - } - return -1; - case 2: - if ((active0 & 0x14380000000L) != 0L) - return 35; - if ((active0 & 0xffebc7fffff00L) != 0L) - { - jjmatchedKind = 54; - jjmatchedPos = 2; - return 35; - } - return -1; - case 3: - if ((active0 & 0x9008070000000L) != 0L) - return 35; - if ((active0 & 0x6fe3c0fffff00L) != 0L) - { - jjmatchedKind = 54; - jjmatchedPos = 3; - return 35; - } - return -1; - case 4: - if ((active0 & 0x23000000100L) != 0L) - return 35; - if ((active0 & 0x6fc0c0ffffe00L) != 0L) - { - jjmatchedKind = 54; - jjmatchedPos = 4; - return 35; - } - return -1; - case 5: - if ((active0 & 0x480c00000000L) != 0L) - return 35; - if ((active0 & 0x6b4000ffffe00L) != 0L) - { - jjmatchedKind = 54; - jjmatchedPos = 5; - return 35; - } - return -1; - case 6: - if ((active0 & 0x6100007bffe00L) != 0L) - { - jjmatchedKind = 54; - jjmatchedPos = 6; - return 35; - } - if ((active0 & 0xa40008400000L) != 0L) - return 35; - return -1; - case 7: - if ((active0 & 0x100007bfee00L) != 0L) - { - jjmatchedKind = 54; - jjmatchedPos = 7; - return 35; - } - if ((active0 & 0x6000000001000L) != 0L) - return 35; - return -1; - case 8: - if ((active0 & 0x3bdec00L) != 0L) - { - jjmatchedKind = 54; - jjmatchedPos = 8; - return 35; - } - if ((active0 & 0x100004020200L) != 0L) - return 35; - return -1; - case 9: - if ((active0 & 0x3bdec00L) != 0L) - { - jjmatchedKind = 54; - jjmatchedPos = 9; - return 35; - } - return -1; - case 10: - if ((active0 & 0x800L) != 0L) - return 35; - if ((active0 & 0x3bde400L) != 0L) - { - jjmatchedKind = 54; - jjmatchedPos = 10; - return 35; - } - return -1; - case 11: - if ((active0 & 0x1846000L) != 0L) - return 35; - if ((active0 & 0x2398400L) != 0L) - { - jjmatchedKind = 54; - jjmatchedPos = 11; - return 35; - } - return -1; - case 12: - if ((active0 & 0x2010400L) != 0L) - return 35; - if ((active0 & 0x388000L) != 0L) - { - jjmatchedKind = 54; - jjmatchedPos = 12; - return 35; - } - return -1; - case 13: - if ((active0 & 0x80000L) != 0L) - return 35; - if ((active0 & 0x308000L) != 0L) - { - jjmatchedKind = 54; - jjmatchedPos = 13; - return 35; - } - return -1; - case 14: - if ((active0 & 0x308000L) != 0L) - { - jjmatchedKind = 54; - jjmatchedPos = 14; - return 35; - } - return -1; - case 15: - if ((active0 & 0x208000L) != 0L) - return 35; - if ((active0 & 0x100000L) != 0L) - { - jjmatchedKind = 54; - jjmatchedPos = 15; - return 35; - } - return -1; - case 16: - if ((active0 & 0x100000L) != 0L) - { - jjmatchedKind = 54; - jjmatchedPos = 16; - return 35; - } - return -1; - default : - return -1; - } -} -private final int jjStartNfa_0(int pos, long active0, long active1) -{ - return jjMoveNfa_0(jjStopStringLiteralDfa_0(pos, active0, active1), pos + 1); -} -private int jjStopAtPos(int pos, int kind) -{ - jjmatchedKind = kind; - jjmatchedPos = pos; - return pos + 1; -} -private int jjMoveStringLiteralDfa0_0() -{ - switch(curChar) - { - case 40: - return jjStopAtPos(0, 67); - case 41: - return jjStopAtPos(0, 68); - case 44: - return jjStopAtPos(0, 59); - case 58: - return jjStopAtPos(0, 66); - case 59: - return jjStopAtPos(0, 60); - case 60: - return jjStopAtPos(0, 69); - case 61: - return jjStopAtPos(0, 63); - case 62: - return jjStopAtPos(0, 70); - case 91: - return jjStopAtPos(0, 64); - case 93: - return jjStopAtPos(0, 65); - case 97: - return jjMoveStringLiteralDfa1_0(0x20000000000L); - case 98: - return jjMoveStringLiteralDfa1_0(0x60000000L); - case 99: - return jjMoveStringLiteralDfa1_0(0xdd00L); - case 100: - return jjMoveStringLiteralDfa1_0(0x400000000L); - case 101: - return jjMoveStringLiteralDfa1_0(0x1300000000000L); - case 105: - return jjMoveStringLiteralDfa1_0(0x388000000L); - case 106: - return jjMoveStringLiteralDfa1_0(0x2000L); - case 108: - return jjMoveStringLiteralDfa1_0(0x8000000000L); - case 109: - return jjMoveStringLiteralDfa1_0(0x4000000000L); - case 110: - return jjMoveStringLiteralDfa1_0(0x200L); - case 111: - return jjMoveStringLiteralDfa1_0(0x4000000000000L); - case 112: - return jjMoveStringLiteralDfa1_0(0x70000L); - case 114: - return jjMoveStringLiteralDfa1_0(0x2000000080000L); - case 115: - return jjMoveStringLiteralDfa1_0(0x8893800300000L); - case 116: - return jjMoveStringLiteralDfa1_0(0x440000000000L); - case 118: - return jjMoveStringLiteralDfa1_0(0x10000000L); - case 120: - return jjMoveStringLiteralDfa1_0(0x7c00000L); - case 123: - return jjStopAtPos(0, 61); - case 125: - return jjStopAtPos(0, 62); - default : - return jjMoveNfa_0(0, 0); - } -} -private int jjMoveStringLiteralDfa1_0(long active0) -{ - try { curChar = input_stream.readChar(); } - catch(java.io.IOException e) { + public void setDebugStream(java.io.PrintStream ds) { + debugStream = ds; + } + + private final int jjStopStringLiteralDfa_0(int pos, long active0, long active1) { + switch (pos) { + case 0: + if ((active0 & 0xfffffffffff00L) != 0L) { + jjmatchedKind = 54; + return 35; + } + return -1; + case 1: + if ((active0 & 0xfffffffffff00L) != 0L) { + jjmatchedKind = 54; + jjmatchedPos = 1; + return 35; + } + return -1; + case 2: + if ((active0 & 0x14380000000L) != 0L) { + return 35; + } + if ((active0 & 0xffebc7fffff00L) != 0L) { + jjmatchedKind = 54; + jjmatchedPos = 2; + return 35; + } + return -1; + case 3: + if ((active0 & 0x9008070000000L) != 0L) { + return 35; + } + if ((active0 & 0x6fe3c0fffff00L) != 0L) { + jjmatchedKind = 54; + jjmatchedPos = 3; + return 35; + } + return -1; + case 4: + if ((active0 & 0x23000000100L) != 0L) { + return 35; + } + if ((active0 & 0x6fc0c0ffffe00L) != 0L) { + jjmatchedKind = 54; + jjmatchedPos = 4; + return 35; + } + return -1; + case 5: + if ((active0 & 0x480c00000000L) != 0L) { + return 35; + } + if ((active0 & 0x6b4000ffffe00L) != 0L) { + jjmatchedKind = 54; + jjmatchedPos = 5; + return 35; + } + return -1; + case 6: + if ((active0 & 0x6100007bffe00L) != 0L) { + jjmatchedKind = 54; + jjmatchedPos = 6; + return 35; + } + if ((active0 & 0xa40008400000L) != 0L) { + return 35; + } + return -1; + case 7: + if ((active0 & 0x100007bfee00L) != 0L) { + jjmatchedKind = 54; + jjmatchedPos = 7; + return 35; + } + if ((active0 & 0x6000000001000L) != 0L) { + return 35; + } + return -1; + case 8: + if ((active0 & 0x3bdec00L) != 0L) { + jjmatchedKind = 54; + jjmatchedPos = 8; + return 35; + } + if ((active0 & 0x100004020200L) != 0L) { + return 35; + } + return -1; + case 9: + if ((active0 & 0x3bdec00L) != 0L) { + jjmatchedKind = 54; + jjmatchedPos = 9; + return 35; + } + return -1; + case 10: + if ((active0 & 0x800L) != 0L) { + return 35; + } + if ((active0 & 0x3bde400L) != 0L) { + jjmatchedKind = 54; + jjmatchedPos = 10; + return 35; + } + return -1; + case 11: + if ((active0 & 0x1846000L) != 0L) { + return 35; + } + if ((active0 & 0x2398400L) != 0L) { + jjmatchedKind = 54; + jjmatchedPos = 11; + return 35; + } + return -1; + case 12: + if ((active0 & 0x2010400L) != 0L) { + return 35; + } + if ((active0 & 0x388000L) != 0L) { + jjmatchedKind = 54; + jjmatchedPos = 12; + return 35; + } + return -1; + case 13: + if ((active0 & 0x80000L) != 0L) { + return 35; + } + if ((active0 & 0x308000L) != 0L) { + jjmatchedKind = 54; + jjmatchedPos = 13; + return 35; + } + return -1; + case 14: + if ((active0 & 0x308000L) != 0L) { + jjmatchedKind = 54; + jjmatchedPos = 14; + return 35; + } + return -1; + case 15: + if ((active0 & 0x208000L) != 0L) { + return 35; + } + if ((active0 & 0x100000L) != 0L) { + jjmatchedKind = 54; + jjmatchedPos = 15; + return 35; + } + return -1; + case 16: + if ((active0 & 0x100000L) != 0L) { + jjmatchedKind = 54; + jjmatchedPos = 16; + return 35; + } + return -1; + default: + return -1; + } + } + + private final int jjStartNfa_0(int pos, long active0, long active1) { + return jjMoveNfa_0(jjStopStringLiteralDfa_0(pos, active0, active1), pos + 1); + } + + private int jjStopAtPos(int pos, int kind) { + jjmatchedKind = kind; + jjmatchedPos = pos; + return pos + 1; + } + + private int jjMoveStringLiteralDfa0_0() { + switch (curChar) { + case 40: + return jjStopAtPos(0, 67); + case 41: + return jjStopAtPos(0, 68); + case 44: + return jjStopAtPos(0, 59); + case 58: + return jjStopAtPos(0, 66); + case 59: + return jjStopAtPos(0, 60); + case 60: + return jjStopAtPos(0, 69); + case 61: + return jjStopAtPos(0, 63); + case 62: + return jjStopAtPos(0, 70); + case 91: + return jjStopAtPos(0, 64); + case 93: + return jjStopAtPos(0, 65); + case 97: + return jjMoveStringLiteralDfa1_0(0x20000000000L); + case 98: + return jjMoveStringLiteralDfa1_0(0x60000000L); + case 99: + return jjMoveStringLiteralDfa1_0(0xdd00L); + case 100: + return jjMoveStringLiteralDfa1_0(0x400000000L); + case 101: + return jjMoveStringLiteralDfa1_0(0x1300000000000L); + case 105: + return jjMoveStringLiteralDfa1_0(0x388000000L); + case 106: + return jjMoveStringLiteralDfa1_0(0x2000L); + case 108: + return jjMoveStringLiteralDfa1_0(0x8000000000L); + case 109: + return jjMoveStringLiteralDfa1_0(0x4000000000L); + case 110: + return jjMoveStringLiteralDfa1_0(0x200L); + case 111: + return jjMoveStringLiteralDfa1_0(0x4000000000000L); + case 112: + return jjMoveStringLiteralDfa1_0(0x70000L); + case 114: + return jjMoveStringLiteralDfa1_0(0x2000000080000L); + case 115: + return jjMoveStringLiteralDfa1_0(0x8893800300000L); + case 116: + return jjMoveStringLiteralDfa1_0(0x440000000000L); + case 118: + return jjMoveStringLiteralDfa1_0(0x10000000L); + case 120: + return jjMoveStringLiteralDfa1_0(0x7c00000L); + case 123: + return jjStopAtPos(0, 61); + case 125: + return jjStopAtPos(0, 62); + default: + return jjMoveNfa_0(0, 0); + } + } + + private int jjMoveStringLiteralDfa1_0(long active0) { + try { + curChar = input_stream.readChar(); + } catch (java.io.IOException e) { jjStopStringLiteralDfa_0(0, active0, 0L); return 1; - } - switch(curChar) - { - case 49: - return jjMoveStringLiteralDfa2_0(active0, 0x80000000L); - case 51: - return jjMoveStringLiteralDfa2_0(active0, 0x100000000L); - case 54: - return jjMoveStringLiteralDfa2_0(active0, 0x200000000L); - case 97: - return jjMoveStringLiteralDfa2_0(active0, 0x4000002200L); - case 101: - return jjMoveStringLiteralDfa2_0(active0, 0x2812000040000L); - case 104: - return jjMoveStringLiteralDfa2_0(active0, 0x400000010000L); - case 105: - return jjMoveStringLiteralDfa2_0(active0, 0x8000000000L); - case 107: - return jjMoveStringLiteralDfa2_0(active0, 0x8000000000000L); - case 108: - return jjMoveStringLiteralDfa2_0(active0, 0x1000000000L); - case 109: - return jjMoveStringLiteralDfa2_0(active0, 0x300000L); - case 110: - return jjMoveStringLiteralDfa2_0(active0, 0x1000008000000L); - case 111: - return jjMoveStringLiteralDfa2_0(active0, 0x430004100L); - case 112: - return jjMoveStringLiteralDfa2_0(active0, 0x4000000001c00L); - case 115: - return jjMoveStringLiteralDfa2_0(active0, 0x20007c08000L); - case 116: - return jjMoveStringLiteralDfa2_0(active0, 0x80800000000L); - case 117: - return jjMoveStringLiteralDfa2_0(active0, 0x80000L); - case 120: - return jjMoveStringLiteralDfa2_0(active0, 0x300000000000L); - case 121: - return jjMoveStringLiteralDfa2_0(active0, 0x40040020000L); - default : - break; - } - return jjStartNfa_0(0, active0, 0L); -} -private int jjMoveStringLiteralDfa2_0(long old0, long active0) -{ - if (((active0 &= old0)) == 0L) + } + switch (curChar) { + case 49: + return jjMoveStringLiteralDfa2_0(active0, 0x80000000L); + case 51: + return jjMoveStringLiteralDfa2_0(active0, 0x100000000L); + case 54: + return jjMoveStringLiteralDfa2_0(active0, 0x200000000L); + case 97: + return jjMoveStringLiteralDfa2_0(active0, 0x4000002200L); + case 101: + return jjMoveStringLiteralDfa2_0(active0, 0x2812000040000L); + case 104: + return jjMoveStringLiteralDfa2_0(active0, 0x400000010000L); + case 105: + return jjMoveStringLiteralDfa2_0(active0, 0x8000000000L); + case 107: + return jjMoveStringLiteralDfa2_0(active0, 0x8000000000000L); + case 108: + return jjMoveStringLiteralDfa2_0(active0, 0x1000000000L); + case 109: + return jjMoveStringLiteralDfa2_0(active0, 0x300000L); + case 110: + return jjMoveStringLiteralDfa2_0(active0, 0x1000008000000L); + case 111: + return jjMoveStringLiteralDfa2_0(active0, 0x430004100L); + case 112: + return jjMoveStringLiteralDfa2_0(active0, 0x4000000001c00L); + case 115: + return jjMoveStringLiteralDfa2_0(active0, 0x20007c08000L); + case 116: + return jjMoveStringLiteralDfa2_0(active0, 0x80800000000L); + case 117: + return jjMoveStringLiteralDfa2_0(active0, 0x80000L); + case 120: + return jjMoveStringLiteralDfa2_0(active0, 0x300000000000L); + case 121: + return jjMoveStringLiteralDfa2_0(active0, 0x40040020000L); + default: + break; + } + return jjStartNfa_0(0, active0, 0L); + } + + private int jjMoveStringLiteralDfa2_0(long old0, long active0) { + if (((active0 &= old0)) == 0L) { return jjStartNfa_0(0, old0, 0L); - try { curChar = input_stream.readChar(); } - catch(java.io.IOException e) { + } + try { + curChar = input_stream.readChar(); + } catch (java.io.IOException e) { jjStopStringLiteralDfa_0(1, active0, 0L); return 2; - } - switch(curChar) - { - case 50: - if ((active0 & 0x100000000L) != 0L) - return jjStartNfaWithStates_0(2, 32, 35); - break; - case 52: - if ((active0 & 0x200000000L) != 0L) - return jjStartNfaWithStates_0(2, 33, 35); - break; - case 54: - if ((active0 & 0x80000000L) != 0L) - return jjStartNfaWithStates_0(2, 31, 35); - break; - case 95: - return jjMoveStringLiteralDfa3_0(active0, 0x20000L); - case 97: - return jjMoveStringLiteralDfa3_0(active0, 0x300000L); - case 98: - return jjMoveStringLiteralDfa3_0(active0, 0x80000L); - case 99: - return jjMoveStringLiteralDfa3_0(active0, 0x100008004000L); - case 100: - return jjMoveStringLiteralDfa3_0(active0, 0x7c00000L); - case 104: - return jjMoveStringLiteralDfa3_0(active0, 0x8000L); - case 105: - return jjMoveStringLiteralDfa3_0(active0, 0x8001010000000L); - case 109: - return jjMoveStringLiteralDfa3_0(active0, 0x200L); - case 110: - return jjMoveStringLiteralDfa3_0(active0, 0x2000000100L); - case 111: - return jjMoveStringLiteralDfa3_0(active0, 0x20000000L); - case 112: - if ((active0 & 0x4000000000L) != 0L) - return jjStartNfaWithStates_0(2, 38, 35); - return jjMoveStringLiteralDfa3_0(active0, 0x40000011c00L); - case 113: - return jjMoveStringLiteralDfa3_0(active0, 0x2000000000000L); - case 114: - return jjMoveStringLiteralDfa3_0(active0, 0xc80800040000L); - case 115: - return jjMoveStringLiteralDfa3_0(active0, 0x8000000000L); - case 116: - if ((active0 & 0x10000000000L) != 0L) - return jjStartNfaWithStates_0(2, 40, 35); - return jjMoveStringLiteralDfa3_0(active0, 0x4200040000000L); - case 117: - return jjMoveStringLiteralDfa3_0(active0, 0x1000400000000L); - case 118: - return jjMoveStringLiteralDfa3_0(active0, 0x2000L); - case 121: - return jjMoveStringLiteralDfa3_0(active0, 0x20000000000L); - default : - break; - } - return jjStartNfa_0(1, active0, 0L); -} -private int jjMoveStringLiteralDfa3_0(long old0, long active0) -{ - if (((active0 &= old0)) == 0L) + } + switch (curChar) { + case 50: + if ((active0 & 0x100000000L) != 0L) { + return jjStartNfaWithStates_0(2, 32, 35); + } + break; + case 52: + if ((active0 & 0x200000000L) != 0L) { + return jjStartNfaWithStates_0(2, 33, 35); + } + break; + case 54: + if ((active0 & 0x80000000L) != 0L) { + return jjStartNfaWithStates_0(2, 31, 35); + } + break; + case 95: + return jjMoveStringLiteralDfa3_0(active0, 0x20000L); + case 97: + return jjMoveStringLiteralDfa3_0(active0, 0x300000L); + case 98: + return jjMoveStringLiteralDfa3_0(active0, 0x80000L); + case 99: + return jjMoveStringLiteralDfa3_0(active0, 0x100008004000L); + case 100: + return jjMoveStringLiteralDfa3_0(active0, 0x7c00000L); + case 104: + return jjMoveStringLiteralDfa3_0(active0, 0x8000L); + case 105: + return jjMoveStringLiteralDfa3_0(active0, 0x8001010000000L); + case 109: + return jjMoveStringLiteralDfa3_0(active0, 0x200L); + case 110: + return jjMoveStringLiteralDfa3_0(active0, 0x2000000100L); + case 111: + return jjMoveStringLiteralDfa3_0(active0, 0x20000000L); + case 112: + if ((active0 & 0x4000000000L) != 0L) { + return jjStartNfaWithStates_0(2, 38, 35); + } + return jjMoveStringLiteralDfa3_0(active0, 0x40000011c00L); + case 113: + return jjMoveStringLiteralDfa3_0(active0, 0x2000000000000L); + case 114: + return jjMoveStringLiteralDfa3_0(active0, 0xc80800040000L); + case 115: + return jjMoveStringLiteralDfa3_0(active0, 0x8000000000L); + case 116: + if ((active0 & 0x10000000000L) != 0L) { + return jjStartNfaWithStates_0(2, 40, 35); + } + return jjMoveStringLiteralDfa3_0(active0, 0x4200040000000L); + case 117: + return jjMoveStringLiteralDfa3_0(active0, 0x1000400000000L); + case 118: + return jjMoveStringLiteralDfa3_0(active0, 0x2000L); + case 121: + return jjMoveStringLiteralDfa3_0(active0, 0x20000000000L); + default: + break; + } + return jjStartNfa_0(1, active0, 0L); + } + + private int jjMoveStringLiteralDfa3_0(long old0, long active0) { + if (((active0 &= old0)) == 0L) { return jjStartNfa_0(1, old0, 0L); - try { curChar = input_stream.readChar(); } - catch(java.io.IOException e) { + } + try { + curChar = input_stream.readChar(); + } catch (java.io.IOException e) { jjStopStringLiteralDfa_0(2, active0, 0L); return 3; - } - switch(curChar) - { - case 95: - return jjMoveStringLiteralDfa4_0(active0, 0x7c11c00L); - case 97: - return jjMoveStringLiteralDfa4_0(active0, 0xa000L); - case 98: - return jjMoveStringLiteralDfa4_0(active0, 0x400000000L); - case 100: - if ((active0 & 0x10000000L) != 0L) - return jjStartNfaWithStates_0(3, 28, 35); - break; - case 101: - if ((active0 & 0x40000000L) != 0L) - return jjStartNfaWithStates_0(3, 30, 35); - return jjMoveStringLiteralDfa4_0(active0, 0x340000000200L); - case 105: - return jjMoveStringLiteralDfa4_0(active0, 0x4000800000000L); - case 108: - if ((active0 & 0x20000000L) != 0L) - return jjStartNfaWithStates_0(3, 29, 35); - return jjMoveStringLiteralDfa4_0(active0, 0x8340000L); - case 109: - if ((active0 & 0x1000000000000L) != 0L) - return jjStartNfaWithStates_0(3, 48, 35); - return jjMoveStringLiteralDfa4_0(active0, 0x20000L); - case 110: - return jjMoveStringLiteralDfa4_0(active0, 0x20000000000L); - case 111: - return jjMoveStringLiteralDfa4_0(active0, 0x400000004000L); - case 112: - if ((active0 & 0x8000000000000L) != 0L) - return jjStartNfaWithStates_0(3, 51, 35); - break; - case 115: - return jjMoveStringLiteralDfa4_0(active0, 0x1000000100L); - case 116: - if ((active0 & 0x8000000000L) != 0L) - return jjStartNfaWithStates_0(3, 39, 35); - break; - case 117: - return jjMoveStringLiteralDfa4_0(active0, 0x2082000000000L); - case 118: - return jjMoveStringLiteralDfa4_0(active0, 0x800000000000L); - case 121: - return jjMoveStringLiteralDfa4_0(active0, 0x80000L); - default : - break; - } - return jjStartNfa_0(2, active0, 0L); -} -private int jjMoveStringLiteralDfa4_0(long old0, long active0) -{ - if (((active0 &= old0)) == 0L) + } + switch (curChar) { + case 95: + return jjMoveStringLiteralDfa4_0(active0, 0x7c11c00L); + case 97: + return jjMoveStringLiteralDfa4_0(active0, 0xa000L); + case 98: + return jjMoveStringLiteralDfa4_0(active0, 0x400000000L); + case 100: + if ((active0 & 0x10000000L) != 0L) { + return jjStartNfaWithStates_0(3, 28, 35); + } + break; + case 101: + if ((active0 & 0x40000000L) != 0L) { + return jjStartNfaWithStates_0(3, 30, 35); + } + return jjMoveStringLiteralDfa4_0(active0, 0x340000000200L); + case 105: + return jjMoveStringLiteralDfa4_0(active0, 0x4000800000000L); + case 108: + if ((active0 & 0x20000000L) != 0L) { + return jjStartNfaWithStates_0(3, 29, 35); + } + return jjMoveStringLiteralDfa4_0(active0, 0x8340000L); + case 109: + if ((active0 & 0x1000000000000L) != 0L) { + return jjStartNfaWithStates_0(3, 48, 35); + } + return jjMoveStringLiteralDfa4_0(active0, 0x20000L); + case 110: + return jjMoveStringLiteralDfa4_0(active0, 0x20000000000L); + case 111: + return jjMoveStringLiteralDfa4_0(active0, 0x400000004000L); + case 112: + if ((active0 & 0x8000000000000L) != 0L) { + return jjStartNfaWithStates_0(3, 51, 35); + } + break; + case 115: + return jjMoveStringLiteralDfa4_0(active0, 0x1000000100L); + case 116: + if ((active0 & 0x8000000000L) != 0L) { + return jjStartNfaWithStates_0(3, 39, 35); + } + break; + case 117: + return jjMoveStringLiteralDfa4_0(active0, 0x2082000000000L); + case 118: + return jjMoveStringLiteralDfa4_0(active0, 0x800000000000L); + case 121: + return jjMoveStringLiteralDfa4_0(active0, 0x80000L); + default: + break; + } + return jjStartNfa_0(2, active0, 0L); + } + + private int jjMoveStringLiteralDfa4_0(long old0, long active0) { + if (((active0 &= old0)) == 0L) { return jjStartNfa_0(2, old0, 0L); - try { curChar = input_stream.readChar(); } - catch(java.io.IOException e) { + } + try { + curChar = input_stream.readChar(); + } catch (java.io.IOException e) { jjStopStringLiteralDfa_0(3, active0, 0L); return 4; - } - switch(curChar) - { - case 95: - return jjMoveStringLiteralDfa5_0(active0, 0xc2000L); - case 97: - return jjMoveStringLiteralDfa5_0(active0, 0x4404000L); - case 99: - if ((active0 & 0x20000000000L) != 0L) - return jjStartNfaWithStates_0(4, 41, 35); - return jjMoveStringLiteralDfa5_0(active0, 0x80000000000L); - case 100: - return jjMoveStringLiteralDfa5_0(active0, 0x40000000000L); - case 105: - return jjMoveStringLiteralDfa5_0(active0, 0x2800000000800L); - case 108: - return jjMoveStringLiteralDfa5_0(active0, 0x400300000L); - case 109: - if ((active0 & 0x2000000000L) != 0L) - return jjStartNfaWithStates_0(4, 37, 35); - break; - case 110: - return jjMoveStringLiteralDfa5_0(active0, 0x200803010400L); - case 111: - return jjMoveStringLiteralDfa5_0(active0, 0x4000000820000L); - case 112: - return jjMoveStringLiteralDfa5_0(active0, 0x100000000000L); - case 114: - return jjMoveStringLiteralDfa5_0(active0, 0x8000L); - case 115: - return jjMoveStringLiteralDfa5_0(active0, 0x200L); - case 116: - if ((active0 & 0x100L) != 0L) - return jjStartNfaWithStates_0(4, 8, 35); - else if ((active0 & 0x1000000000L) != 0L) - return jjStartNfaWithStates_0(4, 36, 35); - return jjMoveStringLiteralDfa5_0(active0, 0x1000L); - case 117: - return jjMoveStringLiteralDfa5_0(active0, 0x8000000L); - case 119: - return jjMoveStringLiteralDfa5_0(active0, 0x400000000000L); - default : - break; - } - return jjStartNfa_0(3, active0, 0L); -} -private int jjMoveStringLiteralDfa5_0(long old0, long active0) -{ - if (((active0 &= old0)) == 0L) + } + switch (curChar) { + case 95: + return jjMoveStringLiteralDfa5_0(active0, 0xc2000L); + case 97: + return jjMoveStringLiteralDfa5_0(active0, 0x4404000L); + case 99: + if ((active0 & 0x20000000000L) != 0L) { + return jjStartNfaWithStates_0(4, 41, 35); + } + return jjMoveStringLiteralDfa5_0(active0, 0x80000000000L); + case 100: + return jjMoveStringLiteralDfa5_0(active0, 0x40000000000L); + case 105: + return jjMoveStringLiteralDfa5_0(active0, 0x2800000000800L); + case 108: + return jjMoveStringLiteralDfa5_0(active0, 0x400300000L); + case 109: + if ((active0 & 0x2000000000L) != 0L) { + return jjStartNfaWithStates_0(4, 37, 35); + } + break; + case 110: + return jjMoveStringLiteralDfa5_0(active0, 0x200803010400L); + case 111: + return jjMoveStringLiteralDfa5_0(active0, 0x4000000820000L); + case 112: + return jjMoveStringLiteralDfa5_0(active0, 0x100000000000L); + case 114: + return jjMoveStringLiteralDfa5_0(active0, 0x8000L); + case 115: + return jjMoveStringLiteralDfa5_0(active0, 0x200L); + case 116: + if ((active0 & 0x100L) != 0L) { + return jjStartNfaWithStates_0(4, 8, 35); + } else if ((active0 & 0x1000000000L) != 0L) { + return jjStartNfaWithStates_0(4, 36, 35); + } + return jjMoveStringLiteralDfa5_0(active0, 0x1000L); + case 117: + return jjMoveStringLiteralDfa5_0(active0, 0x8000000L); + case 119: + return jjMoveStringLiteralDfa5_0(active0, 0x400000000000L); + default: + break; + } + return jjStartNfa_0(3, active0, 0L); + } + + private int jjMoveStringLiteralDfa5_0(long old0, long active0) { + if (((active0 &= old0)) == 0L) { return jjStartNfa_0(3, old0, 0L); - try { curChar = input_stream.readChar(); } - catch(java.io.IOException e) { + } + try { + curChar = input_stream.readChar(); + } catch (java.io.IOException e) { jjStopStringLiteralDfa_0(4, active0, 0L); return 5; - } - switch(curChar) - { - case 95: - return jjMoveStringLiteralDfa6_0(active0, 0x4000L); - case 97: - return jjMoveStringLiteralDfa6_0(active0, 0x2010400L); - case 99: - return jjMoveStringLiteralDfa6_0(active0, 0x800000000000L); - case 100: - return jjMoveStringLiteralDfa6_0(active0, 0x200008020000L); - case 101: - if ((active0 & 0x400000000L) != 0L) - return jjStartNfaWithStates_0(5, 34, 35); - return jjMoveStringLiteralDfa6_0(active0, 0x40000000000L); - case 103: - if ((active0 & 0x800000000L) != 0L) - return jjStartNfaWithStates_0(5, 35, 35); - break; - case 105: - return jjMoveStringLiteralDfa6_0(active0, 0x1000000L); - case 108: - return jjMoveStringLiteralDfa6_0(active0, 0x400000L); - case 110: - return jjMoveStringLiteralDfa6_0(active0, 0x4000000080800L); - case 112: - return jjMoveStringLiteralDfa6_0(active0, 0x84a200L); - case 114: - return jjMoveStringLiteralDfa6_0(active0, 0x2000000000000L); - case 115: - if ((active0 & 0x400000000000L) != 0L) - return jjStartNfaWithStates_0(5, 46, 35); - break; - case 116: - if ((active0 & 0x80000000000L) != 0L) - return jjStartNfaWithStates_0(5, 43, 35); - return jjMoveStringLiteralDfa6_0(active0, 0x100004300000L); - case 121: - return jjMoveStringLiteralDfa6_0(active0, 0x1000L); - default : - break; - } - return jjStartNfa_0(4, active0, 0L); -} -private int jjMoveStringLiteralDfa6_0(long old0, long active0) -{ - if (((active0 &= old0)) == 0L) + } + switch (curChar) { + case 95: + return jjMoveStringLiteralDfa6_0(active0, 0x4000L); + case 97: + return jjMoveStringLiteralDfa6_0(active0, 0x2010400L); + case 99: + return jjMoveStringLiteralDfa6_0(active0, 0x800000000000L); + case 100: + return jjMoveStringLiteralDfa6_0(active0, 0x200008020000L); + case 101: + if ((active0 & 0x400000000L) != 0L) { + return jjStartNfaWithStates_0(5, 34, 35); + } + return jjMoveStringLiteralDfa6_0(active0, 0x40000000000L); + case 103: + if ((active0 & 0x800000000L) != 0L) { + return jjStartNfaWithStates_0(5, 35, 35); + } + break; + case 105: + return jjMoveStringLiteralDfa6_0(active0, 0x1000000L); + case 108: + return jjMoveStringLiteralDfa6_0(active0, 0x400000L); + case 110: + return jjMoveStringLiteralDfa6_0(active0, 0x4000000080800L); + case 112: + return jjMoveStringLiteralDfa6_0(active0, 0x84a200L); + case 114: + return jjMoveStringLiteralDfa6_0(active0, 0x2000000000000L); + case 115: + if ((active0 & 0x400000000000L) != 0L) { + return jjStartNfaWithStates_0(5, 46, 35); + } + break; + case 116: + if ((active0 & 0x80000000000L) != 0L) { + return jjStartNfaWithStates_0(5, 43, 35); + } + return jjMoveStringLiteralDfa6_0(active0, 0x100004300000L); + case 121: + return jjMoveStringLiteralDfa6_0(active0, 0x1000L); + default: + break; + } + return jjStartNfa_0(4, active0, 0L); + } + + private int jjMoveStringLiteralDfa6_0(long old0, long active0) { + if (((active0 &= old0)) == 0L) { return jjStartNfa_0(4, old0, 0L); - try { curChar = input_stream.readChar(); } - catch(java.io.IOException e) { + } + try { + curChar = input_stream.readChar(); + } catch (java.io.IOException e) { jjStopStringLiteralDfa_0(5, active0, 0L); return 6; - } - switch(curChar) - { - case 95: - return jjMoveStringLiteralDfa7_0(active0, 0x8000L); - case 97: - return jjMoveStringLiteralDfa7_0(active0, 0x40000003c2200L); - case 99: - return jjMoveStringLiteralDfa7_0(active0, 0x800L); - case 101: - if ((active0 & 0x8000000L) != 0L) - return jjStartNfaWithStates_0(6, 27, 35); - else if ((active0 & 0x800000000000L) != 0L) - return jjStartNfaWithStates_0(6, 47, 35); - return jjMoveStringLiteralDfa7_0(active0, 0x2000000000000L); - case 102: - if ((active0 & 0x40000000000L) != 0L) - return jjStartNfaWithStates_0(6, 42, 35); - break; - case 105: - return jjMoveStringLiteralDfa7_0(active0, 0x100000000000L); - case 108: - if ((active0 & 0x400000L) != 0L) - return jjStartNfaWithStates_0(6, 22, 35); - return jjMoveStringLiteralDfa7_0(active0, 0x1000000L); - case 109: - return jjMoveStringLiteralDfa7_0(active0, 0x2010400L); - case 112: - return jjMoveStringLiteralDfa7_0(active0, 0x5000L); - case 115: - if ((active0 & 0x200000000000L) != 0L) - return jjStartNfaWithStates_0(6, 45, 35); - break; - case 116: - return jjMoveStringLiteralDfa7_0(active0, 0x4800000L); - case 117: - return jjMoveStringLiteralDfa7_0(active0, 0x20000L); - default : - break; - } - return jjStartNfa_0(5, active0, 0L); -} -private int jjMoveStringLiteralDfa7_0(long old0, long active0) -{ - if (((active0 &= old0)) == 0L) + } + switch (curChar) { + case 95: + return jjMoveStringLiteralDfa7_0(active0, 0x8000L); + case 97: + return jjMoveStringLiteralDfa7_0(active0, 0x40000003c2200L); + case 99: + return jjMoveStringLiteralDfa7_0(active0, 0x800L); + case 101: + if ((active0 & 0x8000000L) != 0L) { + return jjStartNfaWithStates_0(6, 27, 35); + } else if ((active0 & 0x800000000000L) != 0L) { + return jjStartNfaWithStates_0(6, 47, 35); + } + return jjMoveStringLiteralDfa7_0(active0, 0x2000000000000L); + case 102: + if ((active0 & 0x40000000000L) != 0L) { + return jjStartNfaWithStates_0(6, 42, 35); + } + break; + case 105: + return jjMoveStringLiteralDfa7_0(active0, 0x100000000000L); + case 108: + if ((active0 & 0x400000L) != 0L) { + return jjStartNfaWithStates_0(6, 22, 35); + } + return jjMoveStringLiteralDfa7_0(active0, 0x1000000L); + case 109: + return jjMoveStringLiteralDfa7_0(active0, 0x2010400L); + case 112: + return jjMoveStringLiteralDfa7_0(active0, 0x5000L); + case 115: + if ((active0 & 0x200000000000L) != 0L) { + return jjStartNfaWithStates_0(6, 45, 35); + } + break; + case 116: + return jjMoveStringLiteralDfa7_0(active0, 0x4800000L); + case 117: + return jjMoveStringLiteralDfa7_0(active0, 0x20000L); + default: + break; + } + return jjStartNfa_0(5, active0, 0L); + } + + private int jjMoveStringLiteralDfa7_0(long old0, long active0) { + if (((active0 &= old0)) == 0L) { return jjStartNfa_0(5, old0, 0L); - try { curChar = input_stream.readChar(); } - catch(java.io.IOException e) { + } + try { + curChar = input_stream.readChar(); + } catch (java.io.IOException e) { jjStopStringLiteralDfa_0(6, active0, 0L); return 7; - } - switch(curChar) - { - case 99: - return jjMoveStringLiteralDfa8_0(active0, 0x42200L); - case 100: - if ((active0 & 0x2000000000000L) != 0L) - return jjStartNfaWithStates_0(7, 49, 35); - break; - case 101: - if ((active0 & 0x1000L) != 0L) - return jjStartNfaWithStates_0(7, 12, 35); - return jjMoveStringLiteralDfa8_0(active0, 0x2010400L); - case 105: - return jjMoveStringLiteralDfa8_0(active0, 0x800000L); - case 108: - if ((active0 & 0x4000000000000L) != 0L) - return jjStartNfaWithStates_0(7, 50, 35); - return jjMoveStringLiteralDfa8_0(active0, 0x1320800L); - case 109: - return jjMoveStringLiteralDfa8_0(active0, 0x80000L); - case 110: - return jjMoveStringLiteralDfa8_0(active0, 0x8000L); - case 111: - return jjMoveStringLiteralDfa8_0(active0, 0x100000000000L); - case 114: - return jjMoveStringLiteralDfa8_0(active0, 0x4004000L); - default : - break; - } - return jjStartNfa_0(6, active0, 0L); -} -private int jjMoveStringLiteralDfa8_0(long old0, long active0) -{ - if (((active0 &= old0)) == 0L) + } + switch (curChar) { + case 99: + return jjMoveStringLiteralDfa8_0(active0, 0x42200L); + case 100: + if ((active0 & 0x2000000000000L) != 0L) { + return jjStartNfaWithStates_0(7, 49, 35); + } + break; + case 101: + if ((active0 & 0x1000L) != 0L) { + return jjStartNfaWithStates_0(7, 12, 35); + } + return jjMoveStringLiteralDfa8_0(active0, 0x2010400L); + case 105: + return jjMoveStringLiteralDfa8_0(active0, 0x800000L); + case 108: + if ((active0 & 0x4000000000000L) != 0L) { + return jjStartNfaWithStates_0(7, 50, 35); + } + return jjMoveStringLiteralDfa8_0(active0, 0x1320800L); + case 109: + return jjMoveStringLiteralDfa8_0(active0, 0x80000L); + case 110: + return jjMoveStringLiteralDfa8_0(active0, 0x8000L); + case 111: + return jjMoveStringLiteralDfa8_0(active0, 0x100000000000L); + case 114: + return jjMoveStringLiteralDfa8_0(active0, 0x4004000L); + default: + break; + } + return jjStartNfa_0(6, active0, 0L); + } + + private int jjMoveStringLiteralDfa8_0(long old0, long active0) { + if (((active0 &= old0)) == 0L) { return jjStartNfa_0(6, old0, 0L); - try { curChar = input_stream.readChar(); } - catch(java.io.IOException e) { + } + try { + curChar = input_stream.readChar(); + } catch (java.io.IOException e) { jjStopStringLiteralDfa_0(7, active0, 0L); return 8; - } - switch(curChar) - { - case 97: - return jjMoveStringLiteralDfa9_0(active0, 0x1008000L); - case 101: - if ((active0 & 0x200L) != 0L) - return jjStartNfaWithStates_0(8, 9, 35); - else if ((active0 & 0x20000L) != 0L) - return jjStartNfaWithStates_0(8, 17, 35); - return jjMoveStringLiteralDfa9_0(active0, 0x84000L); - case 107: - return jjMoveStringLiteralDfa9_0(active0, 0x342000L); - case 110: - if ((active0 & 0x100000000000L) != 0L) - return jjStartNfaWithStates_0(8, 44, 35); - break; - case 111: - return jjMoveStringLiteralDfa9_0(active0, 0x800000L); - case 115: - if ((active0 & 0x4000000L) != 0L) - return jjStartNfaWithStates_0(8, 26, 35); - return jjMoveStringLiteralDfa9_0(active0, 0x2010400L); - case 117: - return jjMoveStringLiteralDfa9_0(active0, 0x800L); - default : - break; - } - return jjStartNfa_0(7, active0, 0L); -} -private int jjMoveStringLiteralDfa9_0(long old0, long active0) -{ - if (((active0 &= old0)) == 0L) + } + switch (curChar) { + case 97: + return jjMoveStringLiteralDfa9_0(active0, 0x1008000L); + case 101: + if ((active0 & 0x200L) != 0L) { + return jjStartNfaWithStates_0(8, 9, 35); + } else if ((active0 & 0x20000L) != 0L) { + return jjStartNfaWithStates_0(8, 17, 35); + } + return jjMoveStringLiteralDfa9_0(active0, 0x84000L); + case 107: + return jjMoveStringLiteralDfa9_0(active0, 0x342000L); + case 110: + if ((active0 & 0x100000000000L) != 0L) { + return jjStartNfaWithStates_0(8, 44, 35); + } + break; + case 111: + return jjMoveStringLiteralDfa9_0(active0, 0x800000L); + case 115: + if ((active0 & 0x4000000L) != 0L) { + return jjStartNfaWithStates_0(8, 26, 35); + } + return jjMoveStringLiteralDfa9_0(active0, 0x2010400L); + case 117: + return jjMoveStringLiteralDfa9_0(active0, 0x800L); + default: + break; + } + return jjStartNfa_0(7, active0, 0L); + } + + private int jjMoveStringLiteralDfa9_0(long old0, long active0) { + if (((active0 &= old0)) == 0L) { return jjStartNfa_0(7, old0, 0L); - try { curChar = input_stream.readChar(); } - catch(java.io.IOException e) { + } + try { + curChar = input_stream.readChar(); + } catch (java.io.IOException e) { jjStopStringLiteralDfa_0(8, active0, 0L); return 9; - } - switch(curChar) - { - case 95: - return jjMoveStringLiteralDfa10_0(active0, 0x300000L); - case 97: - return jjMoveStringLiteralDfa10_0(active0, 0x42000L); - case 98: - return jjMoveStringLiteralDfa10_0(active0, 0x1000000L); - case 100: - return jjMoveStringLiteralDfa10_0(active0, 0x800L); - case 102: - return jjMoveStringLiteralDfa10_0(active0, 0x4000L); - case 109: - return jjMoveStringLiteralDfa10_0(active0, 0x8000L); - case 110: - return jjMoveStringLiteralDfa10_0(active0, 0x800000L); - case 112: - return jjMoveStringLiteralDfa10_0(active0, 0x2010400L); - case 115: - return jjMoveStringLiteralDfa10_0(active0, 0x80000L); - default : - break; - } - return jjStartNfa_0(8, active0, 0L); -} -private int jjMoveStringLiteralDfa10_0(long old0, long active0) -{ - if (((active0 &= old0)) == 0L) + } + switch (curChar) { + case 95: + return jjMoveStringLiteralDfa10_0(active0, 0x300000L); + case 97: + return jjMoveStringLiteralDfa10_0(active0, 0x42000L); + case 98: + return jjMoveStringLiteralDfa10_0(active0, 0x1000000L); + case 100: + return jjMoveStringLiteralDfa10_0(active0, 0x800L); + case 102: + return jjMoveStringLiteralDfa10_0(active0, 0x4000L); + case 109: + return jjMoveStringLiteralDfa10_0(active0, 0x8000L); + case 110: + return jjMoveStringLiteralDfa10_0(active0, 0x800000L); + case 112: + return jjMoveStringLiteralDfa10_0(active0, 0x2010400L); + case 115: + return jjMoveStringLiteralDfa10_0(active0, 0x80000L); + default: + break; + } + return jjStartNfa_0(8, active0, 0L); + } + + private int jjMoveStringLiteralDfa10_0(long old0, long active0) { + if (((active0 &= old0)) == 0L) { return jjStartNfa_0(8, old0, 0L); - try { curChar = input_stream.readChar(); } - catch(java.io.IOException e) { + } + try { + curChar = input_stream.readChar(); + } catch (java.io.IOException e) { jjStopStringLiteralDfa_0(9, active0, 0L); return 10; - } - switch(curChar) - { - case 97: - return jjMoveStringLiteralDfa11_0(active0, 0x2810400L); - case 99: - return jjMoveStringLiteralDfa11_0(active0, 0x100000L); - case 101: - if ((active0 & 0x800L) != 0L) - return jjStartNfaWithStates_0(10, 11, 35); - return jjMoveStringLiteralDfa11_0(active0, 0x8000L); - case 103: - return jjMoveStringLiteralDfa11_0(active0, 0x42000L); - case 105: - return jjMoveStringLiteralDfa11_0(active0, 0x4000L); - case 108: - return jjMoveStringLiteralDfa11_0(active0, 0x1000000L); - case 112: - return jjMoveStringLiteralDfa11_0(active0, 0x280000L); - default : - break; - } - return jjStartNfa_0(9, active0, 0L); -} -private int jjMoveStringLiteralDfa11_0(long old0, long active0) -{ - if (((active0 &= old0)) == 0L) + } + switch (curChar) { + case 97: + return jjMoveStringLiteralDfa11_0(active0, 0x2810400L); + case 99: + return jjMoveStringLiteralDfa11_0(active0, 0x100000L); + case 101: + if ((active0 & 0x800L) != 0L) { + return jjStartNfaWithStates_0(10, 11, 35); + } + return jjMoveStringLiteralDfa11_0(active0, 0x8000L); + case 103: + return jjMoveStringLiteralDfa11_0(active0, 0x42000L); + case 105: + return jjMoveStringLiteralDfa11_0(active0, 0x4000L); + case 108: + return jjMoveStringLiteralDfa11_0(active0, 0x1000000L); + case 112: + return jjMoveStringLiteralDfa11_0(active0, 0x280000L); + default: + break; + } + return jjStartNfa_0(9, active0, 0L); + } + + private int jjMoveStringLiteralDfa11_0(long old0, long active0) { + if (((active0 &= old0)) == 0L) { return jjStartNfa_0(9, old0, 0L); - try { curChar = input_stream.readChar(); } - catch(java.io.IOException e) { + } + try { + curChar = input_stream.readChar(); + } catch (java.io.IOException e) { jjStopStringLiteralDfa_0(10, active0, 0L); return 11; - } - switch(curChar) - { - case 97: - return jjMoveStringLiteralDfa12_0(active0, 0x180000L); - case 99: - return jjMoveStringLiteralDfa12_0(active0, 0x2010400L); - case 101: - if ((active0 & 0x2000L) != 0L) - return jjStartNfaWithStates_0(11, 13, 35); - else if ((active0 & 0x40000L) != 0L) - return jjStartNfaWithStates_0(11, 18, 35); - else if ((active0 & 0x1000000L) != 0L) - return jjStartNfaWithStates_0(11, 24, 35); - break; - case 108: - if ((active0 & 0x800000L) != 0L) - return jjStartNfaWithStates_0(11, 23, 35); - break; - case 114: - return jjMoveStringLiteralDfa12_0(active0, 0x200000L); - case 115: - return jjMoveStringLiteralDfa12_0(active0, 0x8000L); - case 120: - if ((active0 & 0x4000L) != 0L) - return jjStartNfaWithStates_0(11, 14, 35); - break; - default : - break; - } - return jjStartNfa_0(10, active0, 0L); -} -private int jjMoveStringLiteralDfa12_0(long old0, long active0) -{ - if (((active0 &= old0)) == 0L) + } + switch (curChar) { + case 97: + return jjMoveStringLiteralDfa12_0(active0, 0x180000L); + case 99: + return jjMoveStringLiteralDfa12_0(active0, 0x2010400L); + case 101: + if ((active0 & 0x2000L) != 0L) { + return jjStartNfaWithStates_0(11, 13, 35); + } else if ((active0 & 0x40000L) != 0L) { + return jjStartNfaWithStates_0(11, 18, 35); + } else if ((active0 & 0x1000000L) != 0L) { + return jjStartNfaWithStates_0(11, 24, 35); + } + break; + case 108: + if ((active0 & 0x800000L) != 0L) { + return jjStartNfaWithStates_0(11, 23, 35); + } + break; + case 114: + return jjMoveStringLiteralDfa12_0(active0, 0x200000L); + case 115: + return jjMoveStringLiteralDfa12_0(active0, 0x8000L); + case 120: + if ((active0 & 0x4000L) != 0L) { + return jjStartNfaWithStates_0(11, 14, 35); + } + break; + default: + break; + } + return jjStartNfa_0(10, active0, 0L); + } + + private int jjMoveStringLiteralDfa12_0(long old0, long active0) { + if (((active0 &= old0)) == 0L) { return jjStartNfa_0(10, old0, 0L); - try { curChar = input_stream.readChar(); } - catch(java.io.IOException e) { + } + try { + curChar = input_stream.readChar(); + } catch (java.io.IOException e) { jjStopStringLiteralDfa_0(11, active0, 0L); return 12; - } - switch(curChar) - { - case 99: - return jjMoveStringLiteralDfa13_0(active0, 0x80000L); - case 101: - if ((active0 & 0x400L) != 0L) - return jjStartNfaWithStates_0(12, 10, 35); - else if ((active0 & 0x10000L) != 0L) - return jjStartNfaWithStates_0(12, 16, 35); - else if ((active0 & 0x2000000L) != 0L) - return jjStartNfaWithStates_0(12, 25, 35); - return jjMoveStringLiteralDfa13_0(active0, 0x200000L); - case 112: - return jjMoveStringLiteralDfa13_0(active0, 0x8000L); - case 116: - return jjMoveStringLiteralDfa13_0(active0, 0x100000L); - default : - break; - } - return jjStartNfa_0(11, active0, 0L); -} -private int jjMoveStringLiteralDfa13_0(long old0, long active0) -{ - if (((active0 &= old0)) == 0L) + } + switch (curChar) { + case 99: + return jjMoveStringLiteralDfa13_0(active0, 0x80000L); + case 101: + if ((active0 & 0x400L) != 0L) { + return jjStartNfaWithStates_0(12, 10, 35); + } else if ((active0 & 0x10000L) != 0L) { + return jjStartNfaWithStates_0(12, 16, 35); + } else if ((active0 & 0x2000000L) != 0L) { + return jjStartNfaWithStates_0(12, 25, 35); + } + return jjMoveStringLiteralDfa13_0(active0, 0x200000L); + case 112: + return jjMoveStringLiteralDfa13_0(active0, 0x8000L); + case 116: + return jjMoveStringLiteralDfa13_0(active0, 0x100000L); + default: + break; + } + return jjStartNfa_0(11, active0, 0L); + } + + private int jjMoveStringLiteralDfa13_0(long old0, long active0) { + if (((active0 &= old0)) == 0L) { return jjStartNfa_0(11, old0, 0L); - try { curChar = input_stream.readChar(); } - catch(java.io.IOException e) { + } + try { + curChar = input_stream.readChar(); + } catch (java.io.IOException e) { jjStopStringLiteralDfa_0(12, active0, 0L); return 13; - } - switch(curChar) - { - case 97: - return jjMoveStringLiteralDfa14_0(active0, 0x8000L); - case 101: - if ((active0 & 0x80000L) != 0L) - return jjStartNfaWithStates_0(13, 19, 35); - return jjMoveStringLiteralDfa14_0(active0, 0x100000L); - case 102: - return jjMoveStringLiteralDfa14_0(active0, 0x200000L); - default : - break; - } - return jjStartNfa_0(12, active0, 0L); -} -private int jjMoveStringLiteralDfa14_0(long old0, long active0) -{ - if (((active0 &= old0)) == 0L) + } + switch (curChar) { + case 97: + return jjMoveStringLiteralDfa14_0(active0, 0x8000L); + case 101: + if ((active0 & 0x80000L) != 0L) { + return jjStartNfaWithStates_0(13, 19, 35); + } + return jjMoveStringLiteralDfa14_0(active0, 0x100000L); + case 102: + return jjMoveStringLiteralDfa14_0(active0, 0x200000L); + default: + break; + } + return jjStartNfa_0(12, active0, 0L); + } + + private int jjMoveStringLiteralDfa14_0(long old0, long active0) { + if (((active0 &= old0)) == 0L) { return jjStartNfa_0(12, old0, 0L); - try { curChar = input_stream.readChar(); } - catch(java.io.IOException e) { + } + try { + curChar = input_stream.readChar(); + } catch (java.io.IOException e) { jjStopStringLiteralDfa_0(13, active0, 0L); return 14; - } - switch(curChar) - { - case 99: - return jjMoveStringLiteralDfa15_0(active0, 0x8000L); - case 103: - return jjMoveStringLiteralDfa15_0(active0, 0x100000L); - case 105: - return jjMoveStringLiteralDfa15_0(active0, 0x200000L); - default : - break; - } - return jjStartNfa_0(13, active0, 0L); -} -private int jjMoveStringLiteralDfa15_0(long old0, long active0) -{ - if (((active0 &= old0)) == 0L) + } + switch (curChar) { + case 99: + return jjMoveStringLiteralDfa15_0(active0, 0x8000L); + case 103: + return jjMoveStringLiteralDfa15_0(active0, 0x100000L); + case 105: + return jjMoveStringLiteralDfa15_0(active0, 0x200000L); + default: + break; + } + return jjStartNfa_0(13, active0, 0L); + } + + private int jjMoveStringLiteralDfa15_0(long old0, long active0) { + if (((active0 &= old0)) == 0L) { return jjStartNfa_0(13, old0, 0L); - try { curChar = input_stream.readChar(); } - catch(java.io.IOException e) { + } + try { + curChar = input_stream.readChar(); + } catch (java.io.IOException e) { jjStopStringLiteralDfa_0(14, active0, 0L); return 15; - } - switch(curChar) - { - case 101: - if ((active0 & 0x8000L) != 0L) - return jjStartNfaWithStates_0(15, 15, 35); - break; - case 111: - return jjMoveStringLiteralDfa16_0(active0, 0x100000L); - case 120: - if ((active0 & 0x200000L) != 0L) - return jjStartNfaWithStates_0(15, 21, 35); - break; - default : - break; - } - return jjStartNfa_0(14, active0, 0L); -} -private int jjMoveStringLiteralDfa16_0(long old0, long active0) -{ - if (((active0 &= old0)) == 0L) + } + switch (curChar) { + case 101: + if ((active0 & 0x8000L) != 0L) { + return jjStartNfaWithStates_0(15, 15, 35); + } + break; + case 111: + return jjMoveStringLiteralDfa16_0(active0, 0x100000L); + case 120: + if ((active0 & 0x200000L) != 0L) { + return jjStartNfaWithStates_0(15, 21, 35); + } + break; + default: + break; + } + return jjStartNfa_0(14, active0, 0L); + } + + private int jjMoveStringLiteralDfa16_0(long old0, long active0) { + if (((active0 &= old0)) == 0L) { return jjStartNfa_0(14, old0, 0L); - try { curChar = input_stream.readChar(); } - catch(java.io.IOException e) { + } + try { + curChar = input_stream.readChar(); + } catch (java.io.IOException e) { jjStopStringLiteralDfa_0(15, active0, 0L); return 16; - } - switch(curChar) - { - case 114: - return jjMoveStringLiteralDfa17_0(active0, 0x100000L); - default : - break; - } - return jjStartNfa_0(15, active0, 0L); -} -private int jjMoveStringLiteralDfa17_0(long old0, long active0) -{ - if (((active0 &= old0)) == 0L) + } + switch (curChar) { + case 114: + return jjMoveStringLiteralDfa17_0(active0, 0x100000L); + default: + break; + } + return jjStartNfa_0(15, active0, 0L); + } + + private int jjMoveStringLiteralDfa17_0(long old0, long active0) { + if (((active0 &= old0)) == 0L) { return jjStartNfa_0(15, old0, 0L); - try { curChar = input_stream.readChar(); } - catch(java.io.IOException e) { + } + try { + curChar = input_stream.readChar(); + } catch (java.io.IOException e) { jjStopStringLiteralDfa_0(16, active0, 0L); return 17; - } - switch(curChar) - { - case 121: - if ((active0 & 0x100000L) != 0L) - return jjStartNfaWithStates_0(17, 20, 35); - break; - default : - break; - } - return jjStartNfa_0(16, active0, 0L); -} -private int jjStartNfaWithStates_0(int pos, int kind, int state) -{ - jjmatchedKind = kind; - jjmatchedPos = pos; - try { curChar = input_stream.readChar(); } - catch(java.io.IOException e) { return pos + 1; } - return jjMoveNfa_0(state, pos + 1); -} -static final long[] jjbitVec0 = { - 0x0L, 0x0L, 0xffffffffffffffffL, 0xffffffffffffffffL -}; -private int jjMoveNfa_0(int startState, int curPos) -{ - int startsAt = 0; - jjnewStateCnt = 35; - int i = 1; - jjstateSet[0] = startState; - int kind = 0x7fffffff; - for (;;) - { - if (++jjround == 0x7fffffff) - ReInitRounds(); - if (curChar < 64) - { - long l = 1L << curChar; - do - { - switch(jjstateSet[--i]) - { - case 35: - if ((0x3ff600000000000L & l) != 0L) - { - if (kind > 58) - kind = 58; - jjCheckNAdd(15); - } - if ((0x3ff400000000000L & l) != 0L) - { - if (kind > 54) - kind = 54; - jjCheckNAdd(7); - } - break; - case 0: - if ((0x3ff000000000000L & l) != 0L) - { - if (kind > 54) - kind = 54; - jjCheckNAdd(7); - } - else if ((0x280000000000L & l) != 0L) - jjCheckNAddStates(0, 2); - else if (curChar == 47) - jjAddStates(3, 4); - else if (curChar == 39) - jjCheckNAddTwoStates(12, 13); - else if (curChar == 34) - jjCheckNAddTwoStates(9, 10); - else if (curChar == 35) - jjCheckNAddStates(5, 7); - if ((0x3ff000000000000L & l) != 0L) - { - if (kind > 52) - kind = 52; - jjCheckNAdd(5); - } - else if (curChar == 45) - { - if (kind > 58) - kind = 58; - jjCheckNAdd(15); - } - break; - case 1: - if ((0xfffffffffffffbffL & l) != 0L) - jjCheckNAddStates(5, 7); - break; - case 2: - if ((0x2400L & l) != 0L && kind > 5) - kind = 5; - break; - case 3: - if (curChar == 10 && kind > 5) - kind = 5; - break; - case 4: - if (curChar == 13) - jjstateSet[jjnewStateCnt++] = 3; - break; - case 5: - if ((0x3ff000000000000L & l) == 0L) - break; - if (kind > 52) - kind = 52; - jjCheckNAdd(5); - break; - case 6: - if ((0x3ff000000000000L & l) == 0L) - break; - if (kind > 54) - kind = 54; - jjCheckNAdd(7); - break; - case 7: - if ((0x3ff400000000000L & l) == 0L) - break; - if (kind > 54) - kind = 54; - jjCheckNAdd(7); - break; - case 8: - if (curChar == 34) - jjCheckNAddTwoStates(9, 10); - break; - case 9: - if ((0xfffffffbffffffffL & l) != 0L) - jjCheckNAddTwoStates(9, 10); - break; - case 10: - if (curChar == 34 && kind > 57) - kind = 57; - break; - case 11: - if (curChar == 39) - jjCheckNAddTwoStates(12, 13); - break; - case 12: - if ((0xffffff7fffffffffL & l) != 0L) - jjCheckNAddTwoStates(12, 13); - break; - case 13: - if (curChar == 39 && kind > 57) - kind = 57; - break; - case 14: - if (curChar != 45) - break; - if (kind > 58) - kind = 58; - jjCheckNAdd(15); - break; - case 15: - if ((0x3ff600000000000L & l) == 0L) - break; - if (kind > 58) - kind = 58; - jjCheckNAdd(15); - break; - case 16: - if (curChar == 47) - jjAddStates(3, 4); - break; - case 17: - if (curChar == 47) - jjCheckNAddStates(8, 10); - break; - case 18: - if ((0xffffffffffffdbffL & l) != 0L) - jjCheckNAddStates(8, 10); - break; - case 19: - if ((0x2400L & l) != 0L && kind > 6) - kind = 6; - break; - case 20: - if (curChar == 10 && kind > 6) - kind = 6; - break; - case 21: - if (curChar == 13) - jjstateSet[jjnewStateCnt++] = 20; - break; - case 22: - if (curChar == 42) - jjCheckNAddTwoStates(23, 24); - break; - case 23: - if ((0xfffffbffffffffffL & l) != 0L) - jjCheckNAddTwoStates(23, 24); - break; - case 24: - if (curChar == 42) - jjAddStates(11, 12); - break; - case 25: - if ((0xffff7fffffffffffL & l) != 0L) - jjCheckNAddTwoStates(26, 24); - break; - case 26: - if ((0xfffffbffffffffffL & l) != 0L) - jjCheckNAddTwoStates(26, 24); - break; - case 27: - if (curChar == 47 && kind > 7) - kind = 7; - break; - case 28: - if ((0x280000000000L & l) != 0L) - jjCheckNAddStates(0, 2); - break; - case 29: - if ((0x3ff000000000000L & l) != 0L) - jjCheckNAddTwoStates(29, 30); - break; - case 30: - if (curChar == 46) - jjCheckNAdd(31); - break; - case 31: - if ((0x3ff000000000000L & l) == 0L) - break; - if (kind > 53) - kind = 53; - jjCheckNAddTwoStates(31, 32); - break; - case 33: - if ((0x280000000000L & l) != 0L) - jjCheckNAdd(34); - break; - case 34: - if ((0x3ff000000000000L & l) == 0L) - break; - if (kind > 53) - kind = 53; - jjCheckNAdd(34); - break; - default : break; - } - } while(i != startsAt); + } + switch (curChar) { + case 121: + if ((active0 & 0x100000L) != 0L) { + return jjStartNfaWithStates_0(17, 20, 35); } - else if (curChar < 128) - { - long l = 1L << (curChar & 077); - do - { - switch(jjstateSet[--i]) - { - case 35: - if ((0x7fffffe87fffffeL & l) != 0L) - { - if (kind > 58) - kind = 58; - jjCheckNAdd(15); - } - if ((0x7fffffe87fffffeL & l) != 0L) - { - if (kind > 54) - kind = 54; - jjCheckNAdd(7); - } - break; - case 0: - if ((0x7fffffe87fffffeL & l) != 0L) - { - if (kind > 54) - kind = 54; - jjCheckNAdd(7); - } - if ((0x7fffffe07fffffeL & l) != 0L) - { - if (kind > 58) - kind = 58; - jjCheckNAdd(15); - } - break; - case 1: - jjAddStates(5, 7); - break; - case 6: - if ((0x7fffffe87fffffeL & l) == 0L) - break; - if (kind > 54) - kind = 54; - jjCheckNAdd(7); - break; - case 7: - if ((0x7fffffe87fffffeL & l) == 0L) - break; - if (kind > 54) - kind = 54; - jjCheckNAdd(7); - break; - case 9: - jjAddStates(13, 14); - break; - case 12: - jjAddStates(15, 16); - break; - case 14: - if ((0x7fffffe07fffffeL & l) == 0L) - break; - if (kind > 58) - kind = 58; - jjCheckNAdd(15); - break; - case 15: - if ((0x7fffffe87fffffeL & l) == 0L) - break; - if (kind > 58) - kind = 58; - jjCheckNAdd(15); - break; - case 18: - jjAddStates(8, 10); - break; - case 23: - jjCheckNAddTwoStates(23, 24); - break; - case 25: - case 26: - jjCheckNAddTwoStates(26, 24); - break; - case 32: - if ((0x2000000020L & l) != 0L) - jjAddStates(17, 18); - break; - default : break; - } - } while(i != startsAt); + break; + default: + break; + } + return jjStartNfa_0(16, active0, 0L); + } + + private int jjStartNfaWithStates_0(int pos, int kind, int state) { + jjmatchedKind = kind; + jjmatchedPos = pos; + try { + curChar = input_stream.readChar(); + } catch (java.io.IOException e) { + return pos + 1; + } + return jjMoveNfa_0(state, pos + 1); + } + + static final long[] jjbitVec0 = { 0x0L, 0x0L, 0xffffffffffffffffL, + 0xffffffffffffffffL }; + + private int jjMoveNfa_0(int startState, int curPos) { + int startsAt = 0; + jjnewStateCnt = 35; + int i = 1; + jjstateSet[0] = startState; + int kind = 0x7fffffff; + for (;;) { + if (++jjround == 0x7fffffff) { + ReInitRounds(); } - else - { - int i2 = (curChar & 0xff) >> 6; - long l2 = 1L << (curChar & 077); - do - { - switch(jjstateSet[--i]) - { - case 1: - if ((jjbitVec0[i2] & l2) != 0L) - jjAddStates(5, 7); - break; - case 9: - if ((jjbitVec0[i2] & l2) != 0L) - jjAddStates(13, 14); - break; - case 12: - if ((jjbitVec0[i2] & l2) != 0L) - jjAddStates(15, 16); - break; - case 18: - if ((jjbitVec0[i2] & l2) != 0L) - jjAddStates(8, 10); - break; - case 23: - if ((jjbitVec0[i2] & l2) != 0L) - jjCheckNAddTwoStates(23, 24); - break; - case 25: - case 26: - if ((jjbitVec0[i2] & l2) != 0L) - jjCheckNAddTwoStates(26, 24); - break; - default : break; + if (curChar < 64) { + long l = 1L << curChar; + do { + switch (jjstateSet[--i]) { + case 35: + if ((0x3ff600000000000L & l) != 0L) { + if (kind > 58) { + kind = 58; + } + jjCheckNAdd(15); } - } while(i != startsAt); + if ((0x3ff400000000000L & l) != 0L) { + if (kind > 54) { + kind = 54; + } + jjCheckNAdd(7); + } + break; + case 0: + if ((0x3ff000000000000L & l) != 0L) { + if (kind > 54) { + kind = 54; + } + jjCheckNAdd(7); + } else if ((0x280000000000L & l) != 0L) { + jjCheckNAddStates(0, 2); + } else if (curChar == 47) { + jjAddStates(3, 4); + } else if (curChar == 39) { + jjCheckNAddTwoStates(12, 13); + } else if (curChar == 34) { + jjCheckNAddTwoStates(9, 10); + } else if (curChar == 35) { + jjCheckNAddStates(5, 7); + } + if ((0x3ff000000000000L & l) != 0L) { + if (kind > 52) { + kind = 52; + } + jjCheckNAdd(5); + } else if (curChar == 45) { + if (kind > 58) { + kind = 58; + } + jjCheckNAdd(15); + } + break; + case 1: + if ((0xfffffffffffffbffL & l) != 0L) { + jjCheckNAddStates(5, 7); + } + break; + case 2: + if ((0x2400L & l) != 0L && kind > 5) { + kind = 5; + } + break; + case 3: + if (curChar == 10 && kind > 5) { + kind = 5; + } + break; + case 4: + if (curChar == 13) { + jjstateSet[jjnewStateCnt++] = 3; + } + break; + case 5: + if ((0x3ff000000000000L & l) == 0L) { + break; + } + if (kind > 52) { + kind = 52; + } + jjCheckNAdd(5); + break; + case 6: + if ((0x3ff000000000000L & l) == 0L) { + break; + } + if (kind > 54) { + kind = 54; + } + jjCheckNAdd(7); + break; + case 7: + if ((0x3ff400000000000L & l) == 0L) { + break; + } + if (kind > 54) { + kind = 54; + } + jjCheckNAdd(7); + break; + case 8: + if (curChar == 34) { + jjCheckNAddTwoStates(9, 10); + } + break; + case 9: + if ((0xfffffffbffffffffL & l) != 0L) { + jjCheckNAddTwoStates(9, 10); + } + break; + case 10: + if (curChar == 34 && kind > 57) { + kind = 57; + } + break; + case 11: + if (curChar == 39) { + jjCheckNAddTwoStates(12, 13); + } + break; + case 12: + if ((0xffffff7fffffffffL & l) != 0L) { + jjCheckNAddTwoStates(12, 13); + } + break; + case 13: + if (curChar == 39 && kind > 57) { + kind = 57; + } + break; + case 14: + if (curChar != 45) { + break; + } + if (kind > 58) { + kind = 58; + } + jjCheckNAdd(15); + break; + case 15: + if ((0x3ff600000000000L & l) == 0L) { + break; + } + if (kind > 58) { + kind = 58; + } + jjCheckNAdd(15); + break; + case 16: + if (curChar == 47) { + jjAddStates(3, 4); + } + break; + case 17: + if (curChar == 47) { + jjCheckNAddStates(8, 10); + } + break; + case 18: + if ((0xffffffffffffdbffL & l) != 0L) { + jjCheckNAddStates(8, 10); + } + break; + case 19: + if ((0x2400L & l) != 0L && kind > 6) { + kind = 6; + } + break; + case 20: + if (curChar == 10 && kind > 6) { + kind = 6; + } + break; + case 21: + if (curChar == 13) { + jjstateSet[jjnewStateCnt++] = 20; + } + break; + case 22: + if (curChar == 42) { + jjCheckNAddTwoStates(23, 24); + } + break; + case 23: + if ((0xfffffbffffffffffL & l) != 0L) { + jjCheckNAddTwoStates(23, 24); + } + break; + case 24: + if (curChar == 42) { + jjAddStates(11, 12); + } + break; + case 25: + if ((0xffff7fffffffffffL & l) != 0L) { + jjCheckNAddTwoStates(26, 24); + } + break; + case 26: + if ((0xfffffbffffffffffL & l) != 0L) { + jjCheckNAddTwoStates(26, 24); + } + break; + case 27: + if (curChar == 47 && kind > 7) { + kind = 7; + } + break; + case 28: + if ((0x280000000000L & l) != 0L) { + jjCheckNAddStates(0, 2); + } + break; + case 29: + if ((0x3ff000000000000L & l) != 0L) { + jjCheckNAddTwoStates(29, 30); + } + break; + case 30: + if (curChar == 46) { + jjCheckNAdd(31); + } + break; + case 31: + if ((0x3ff000000000000L & l) == 0L) { + break; + } + if (kind > 53) { + kind = 53; + } + jjCheckNAddTwoStates(31, 32); + break; + case 33: + if ((0x280000000000L & l) != 0L) { + jjCheckNAdd(34); + } + break; + case 34: + if ((0x3ff000000000000L & l) == 0L) { + break; + } + if (kind > 53) { + kind = 53; + } + jjCheckNAdd(34); + break; + default: + break; + } + } while (i != startsAt); + } else if (curChar < 128) { + long l = 1L << (curChar & 077); + do { + switch (jjstateSet[--i]) { + case 35: + if ((0x7fffffe87fffffeL & l) != 0L) { + if (kind > 58) { + kind = 58; + } + jjCheckNAdd(15); + } + if ((0x7fffffe87fffffeL & l) != 0L) { + if (kind > 54) { + kind = 54; + } + jjCheckNAdd(7); + } + break; + case 0: + if ((0x7fffffe87fffffeL & l) != 0L) { + if (kind > 54) { + kind = 54; + } + jjCheckNAdd(7); + } + if ((0x7fffffe07fffffeL & l) != 0L) { + if (kind > 58) { + kind = 58; + } + jjCheckNAdd(15); + } + break; + case 1: + jjAddStates(5, 7); + break; + case 6: + if ((0x7fffffe87fffffeL & l) == 0L) { + break; + } + if (kind > 54) { + kind = 54; + } + jjCheckNAdd(7); + break; + case 7: + if ((0x7fffffe87fffffeL & l) == 0L) { + break; + } + if (kind > 54) { + kind = 54; + } + jjCheckNAdd(7); + break; + case 9: + jjAddStates(13, 14); + break; + case 12: + jjAddStates(15, 16); + break; + case 14: + if ((0x7fffffe07fffffeL & l) == 0L) { + break; + } + if (kind > 58) { + kind = 58; + } + jjCheckNAdd(15); + break; + case 15: + if ((0x7fffffe87fffffeL & l) == 0L) { + break; + } + if (kind > 58) { + kind = 58; + } + jjCheckNAdd(15); + break; + case 18: + jjAddStates(8, 10); + break; + case 23: + jjCheckNAddTwoStates(23, 24); + break; + case 25: + case 26: + jjCheckNAddTwoStates(26, 24); + break; + case 32: + if ((0x2000000020L & l) != 0L) { + jjAddStates(17, 18); + } + break; + default: + break; + } + } while (i != startsAt); + } else { + int i2 = (curChar & 0xff) >> 6; + long l2 = 1L << (curChar & 077); + do { + switch (jjstateSet[--i]) { + case 1: + if ((jjbitVec0[i2] & l2) != 0L) { + jjAddStates(5, 7); + } + break; + case 9: + if ((jjbitVec0[i2] & l2) != 0L) { + jjAddStates(13, 14); + } + break; + case 12: + if ((jjbitVec0[i2] & l2) != 0L) { + jjAddStates(15, 16); + } + break; + case 18: + if ((jjbitVec0[i2] & l2) != 0L) { + jjAddStates(8, 10); + } + break; + case 23: + if ((jjbitVec0[i2] & l2) != 0L) { + jjCheckNAddTwoStates(23, 24); + } + break; + case 25: + case 26: + if ((jjbitVec0[i2] & l2) != 0L) { + jjCheckNAddTwoStates(26, 24); + } + break; + default: + break; + } + } while (i != startsAt); } - if (kind != 0x7fffffff) - { - jjmatchedKind = kind; - jjmatchedPos = curPos; - kind = 0x7fffffff; + if (kind != 0x7fffffff) { + jjmatchedKind = kind; + jjmatchedPos = curPos; + kind = 0x7fffffff; } ++curPos; - if ((i = jjnewStateCnt) == (startsAt = 35 - (jjnewStateCnt = startsAt))) - return curPos; - try { curChar = input_stream.readChar(); } - catch(java.io.IOException e) { return curPos; } - } -} -static final int[] jjnextStates = { - 5, 29, 30, 17, 22, 1, 2, 4, 18, 19, 21, 25, 27, 9, 10, 12, - 13, 33, 34, -}; + if ((i = jjnewStateCnt) == (startsAt = 35 - (jjnewStateCnt = startsAt))) { + return curPos; + } + try { + curChar = input_stream.readChar(); + } catch (java.io.IOException e) { + return curPos; + } + } + } -/** Token literal values. */ -public static final String[] jjstrLiteralImages = { -"", null, null, null, null, null, null, null, "\143\157\156\163\164", -"\156\141\155\145\163\160\141\143\145", "\143\160\160\137\156\141\155\145\163\160\141\143\145", -"\143\160\160\137\151\156\143\154\165\144\145", "\143\160\160\137\164\171\160\145", -"\152\141\166\141\137\160\141\143\153\141\147\145", "\143\157\143\157\141\137\160\162\145\146\151\170", -"\143\163\150\141\162\160\137\156\141\155\145\163\160\141\143\145", "\160\150\160\137\156\141\155\145\163\160\141\143\145", -"\160\171\137\155\157\144\165\154\145", "\160\145\162\154\137\160\141\143\153\141\147\145", -"\162\165\142\171\137\156\141\155\145\163\160\141\143\145", "\163\155\141\154\154\164\141\154\153\137\143\141\164\145\147\157\162\171", -"\163\155\141\154\154\164\141\154\153\137\160\162\145\146\151\170", "\170\163\144\137\141\154\154", -"\170\163\144\137\157\160\164\151\157\156\141\154", "\170\163\144\137\156\151\154\154\141\142\154\145", -"\170\163\144\137\156\141\155\145\163\160\141\143\145", "\170\163\144\137\141\164\164\162\163", "\151\156\143\154\165\144\145", -"\166\157\151\144", "\142\157\157\154", "\142\171\164\145", "\151\61\66", "\151\63\62", -"\151\66\64", "\144\157\165\142\154\145", "\163\164\162\151\156\147", -"\163\154\151\163\164", "\163\145\156\165\155", "\155\141\160", "\154\151\163\164", "\163\145\164", -"\141\163\171\156\143", "\164\171\160\145\144\145\146", "\163\164\162\165\143\164", -"\145\170\143\145\160\164\151\157\156", "\145\170\164\145\156\144\163", "\164\150\162\157\167\163", -"\163\145\162\166\151\143\145", "\145\156\165\155", "\162\145\161\165\151\162\145\144", -"\157\160\164\151\157\156\141\154", "\163\153\151\160", null, null, null, null, null, null, null, "\54", "\73", -"\173", "\175", "\75", "\133", "\135", "\72", "\50", "\51", "\74", "\76", }; + static final int[] jjnextStates = { 5, 29, 30, 17, 22, 1, 2, 4, 18, 19, 21, + 25, 27, 9, 10, 12, 13, 33, 34, }; -/** Lexer state names. */ -public static final String[] lexStateNames = { - "DEFAULT", -}; -static final long[] jjtoToken = { - 0xfe7fffffffffff01L, 0x7fL, -}; -static final long[] jjtoSkip = { - 0xfeL, 0x0L, -}; -protected SimpleCharStream input_stream; -private final int[] jjrounds = new int[35]; -private final int[] jjstateSet = new int[70]; -protected char curChar; -/** Constructor. */ -public thrift_grammarTokenManager(SimpleCharStream stream){ - if (SimpleCharStream.staticFlag) - throw new Error("ERROR: Cannot use a static CharStream class with a non-static lexical analyzer."); - input_stream = stream; -} + /** Token literal values. */ + public static final String[] jjstrLiteralImages = { + "", + null, + null, + null, + null, + null, + null, + null, + "\143\157\156\163\164", + "\156\141\155\145\163\160\141\143\145", + "\143\160\160\137\156\141\155\145\163\160\141\143\145", + "\143\160\160\137\151\156\143\154\165\144\145", + "\143\160\160\137\164\171\160\145", + "\152\141\166\141\137\160\141\143\153\141\147\145", + "\143\157\143\157\141\137\160\162\145\146\151\170", + "\143\163\150\141\162\160\137\156\141\155\145\163\160\141\143\145", + "\160\150\160\137\156\141\155\145\163\160\141\143\145", + "\160\171\137\155\157\144\165\154\145", + "\160\145\162\154\137\160\141\143\153\141\147\145", + "\162\165\142\171\137\156\141\155\145\163\160\141\143\145", + "\163\155\141\154\154\164\141\154\153\137\143\141\164\145\147\157\162\171", + "\163\155\141\154\154\164\141\154\153\137\160\162\145\146\151\170", + "\170\163\144\137\141\154\154", + "\170\163\144\137\157\160\164\151\157\156\141\154", + "\170\163\144\137\156\151\154\154\141\142\154\145", + "\170\163\144\137\156\141\155\145\163\160\141\143\145", + "\170\163\144\137\141\164\164\162\163", "\151\156\143\154\165\144\145", + "\166\157\151\144", "\142\157\157\154", "\142\171\164\145", "\151\61\66", + "\151\63\62", "\151\66\64", "\144\157\165\142\154\145", + "\163\164\162\151\156\147", "\163\154\151\163\164", + "\163\145\156\165\155", "\155\141\160", "\154\151\163\164", + "\163\145\164", "\141\163\171\156\143", "\164\171\160\145\144\145\146", + "\163\164\162\165\143\164", "\145\170\143\145\160\164\151\157\156", + "\145\170\164\145\156\144\163", "\164\150\162\157\167\163", + "\163\145\162\166\151\143\145", "\145\156\165\155", + "\162\145\161\165\151\162\145\144", "\157\160\164\151\157\156\141\154", + "\163\153\151\160", null, null, null, null, null, null, null, "\54", + "\73", "\173", "\175", "\75", "\133", "\135", "\72", "\50", "\51", "\74", + "\76", }; -/** Constructor. */ -public thrift_grammarTokenManager(SimpleCharStream stream, int lexState){ - this(stream); - SwitchTo(lexState); -} + /** Lexer state names. */ + public static final String[] lexStateNames = { "DEFAULT", }; + static final long[] jjtoToken = { 0xfe7fffffffffff01L, 0x7fL, }; + static final long[] jjtoSkip = { 0xfeL, 0x0L, }; + protected SimpleCharStream input_stream; + private final int[] jjrounds = new int[35]; + private final int[] jjstateSet = new int[70]; + protected char curChar; -/** Reinitialise parser. */ -public void ReInit(SimpleCharStream stream) -{ - jjmatchedPos = jjnewStateCnt = 0; - curLexState = defaultLexState; - input_stream = stream; - ReInitRounds(); -} -private void ReInitRounds() -{ - int i; - jjround = 0x80000001; - for (i = 35; i-- > 0;) + /** Constructor. */ + public thrift_grammarTokenManager(SimpleCharStream stream) { + if (SimpleCharStream.staticFlag) { + throw new Error( + "ERROR: Cannot use a static CharStream class with a non-static lexical analyzer."); + } + input_stream = stream; + } + + /** Constructor. */ + public thrift_grammarTokenManager(SimpleCharStream stream, int lexState) { + this(stream); + SwitchTo(lexState); + } + + /** Reinitialise parser. */ + public void ReInit(SimpleCharStream stream) { + jjmatchedPos = jjnewStateCnt = 0; + curLexState = defaultLexState; + input_stream = stream; + ReInitRounds(); + } + + private void ReInitRounds() { + int i; + jjround = 0x80000001; + for (i = 35; i-- > 0;) { jjrounds[i] = 0x80000000; -} + } + } -/** Reinitialise parser. */ -public void ReInit(SimpleCharStream stream, int lexState) -{ - ReInit(stream); - SwitchTo(lexState); -} + /** Reinitialise parser. */ + public void ReInit(SimpleCharStream stream, int lexState) { + ReInit(stream); + SwitchTo(lexState); + } -/** Switch to specified lex state. */ -public void SwitchTo(int lexState) -{ - if (lexState >= 1 || lexState < 0) - throw new TokenMgrError("Error: Ignoring invalid lexical state : " + lexState + ". State unchanged.", TokenMgrError.INVALID_LEXICAL_STATE); - else + /** Switch to specified lex state. */ + public void SwitchTo(int lexState) { + if (lexState >= 1 || lexState < 0) { + throw new TokenMgrError("Error: Ignoring invalid lexical state : " + + lexState + ". State unchanged.", + TokenMgrError.INVALID_LEXICAL_STATE); + } else { curLexState = lexState; -} + } + } -protected Token jjFillToken() -{ - final Token t; - final String curTokenImage; - final int beginLine; - final int endLine; - final int beginColumn; - final int endColumn; - String im = jjstrLiteralImages[jjmatchedKind]; - curTokenImage = (im == null) ? input_stream.GetImage() : im; - beginLine = input_stream.getBeginLine(); - beginColumn = input_stream.getBeginColumn(); - endLine = input_stream.getEndLine(); - endColumn = input_stream.getEndColumn(); - t = Token.newToken(jjmatchedKind); - t.kind = jjmatchedKind; - t.image = curTokenImage; + protected Token jjFillToken() { + final Token t; + final String curTokenImage; + final int beginLine; + final int endLine; + final int beginColumn; + final int endColumn; + String im = jjstrLiteralImages[jjmatchedKind]; + curTokenImage = (im == null) ? input_stream.GetImage() : im; + beginLine = input_stream.getBeginLine(); + beginColumn = input_stream.getBeginColumn(); + endLine = input_stream.getEndLine(); + endColumn = input_stream.getEndColumn(); + t = Token.newToken(jjmatchedKind); + t.kind = jjmatchedKind; + t.image = curTokenImage; - t.beginLine = beginLine; - t.endLine = endLine; - t.beginColumn = beginColumn; - t.endColumn = endColumn; + t.beginLine = beginLine; + t.endLine = endLine; + t.beginColumn = beginColumn; + t.endColumn = endColumn; - return t; -} + return t; + } -int curLexState = 0; -int defaultLexState = 0; -int jjnewStateCnt; -int jjround; -int jjmatchedPos; -int jjmatchedKind; + int curLexState = 0; + int defaultLexState = 0; + int jjnewStateCnt; + int jjround; + int jjmatchedPos; + int jjmatchedKind; -/** Get the next Token. */ -public Token getNextToken() -{ - Token matchedToken; - int curPos = 0; + /** Get the next Token. */ + public Token getNextToken() { + Token matchedToken; + int curPos = 0; - EOFLoop : - for (;;) - { - try - { - curChar = input_stream.BeginToken(); - } - catch(java.io.IOException e) - { - jjmatchedKind = 0; - matchedToken = jjFillToken(); - return matchedToken; - } + EOFLoop: for (;;) { + try { + curChar = input_stream.BeginToken(); + } catch (java.io.IOException e) { + jjmatchedKind = 0; + matchedToken = jjFillToken(); + return matchedToken; + } - try { input_stream.backup(0); - while (curChar <= 32 && (0x100002600L & (1L << curChar)) != 0L) - curChar = input_stream.BeginToken(); - } - catch (java.io.IOException e1) { continue EOFLoop; } - jjmatchedKind = 0x7fffffff; - jjmatchedPos = 0; - curPos = jjMoveStringLiteralDfa0_0(); - if (jjmatchedKind != 0x7fffffff) - { - if (jjmatchedPos + 1 < curPos) - input_stream.backup(curPos - jjmatchedPos - 1); - if ((jjtoToken[jjmatchedKind >> 6] & (1L << (jjmatchedKind & 077))) != 0L) - { - matchedToken = jjFillToken(); - return matchedToken; + try { + input_stream.backup(0); + while (curChar <= 32 && (0x100002600L & (1L << curChar)) != 0L) { + curChar = input_stream.BeginToken(); + } + } catch (java.io.IOException e1) { + continue EOFLoop; } - else - { - continue EOFLoop; + jjmatchedKind = 0x7fffffff; + jjmatchedPos = 0; + curPos = jjMoveStringLiteralDfa0_0(); + if (jjmatchedKind != 0x7fffffff) { + if (jjmatchedPos + 1 < curPos) { + input_stream.backup(curPos - jjmatchedPos - 1); + } + if ((jjtoToken[jjmatchedKind >> 6] & (1L << (jjmatchedKind & 077))) != 0L) { + matchedToken = jjFillToken(); + return matchedToken; + } else { + continue EOFLoop; + } } - } - int error_line = input_stream.getEndLine(); - int error_column = input_stream.getEndColumn(); - String error_after = null; - boolean EOFSeen = false; - try { input_stream.readChar(); input_stream.backup(1); } - catch (java.io.IOException e1) { - EOFSeen = true; - error_after = curPos <= 1 ? "" : input_stream.GetImage(); - if (curChar == '\n' || curChar == '\r') { - error_line++; - error_column = 0; + int error_line = input_stream.getEndLine(); + int error_column = input_stream.getEndColumn(); + String error_after = null; + boolean EOFSeen = false; + try { + input_stream.readChar(); + input_stream.backup(1); + } catch (java.io.IOException e1) { + EOFSeen = true; + error_after = curPos <= 1 ? "" : input_stream.GetImage(); + if (curChar == '\n' || curChar == '\r') { + error_line++; + error_column = 0; + } else { + error_column++; + } } - else - error_column++; - } - if (!EOFSeen) { - input_stream.backup(1); - error_after = curPos <= 1 ? "" : input_stream.GetImage(); - } - throw new TokenMgrError(EOFSeen, curLexState, error_line, error_column, error_after, curChar, TokenMgrError.LEXICAL_ERROR); + if (!EOFSeen) { + input_stream.backup(1); + error_after = curPos <= 1 ? "" : input_stream.GetImage(); + } + throw new TokenMgrError(EOFSeen, curLexState, error_line, error_column, + error_after, curChar, TokenMgrError.LEXICAL_ERROR); + } } -} -private void jjCheckNAdd(int state) -{ - if (jjrounds[state] != jjround) - { + private void jjCheckNAdd(int state) { + if (jjrounds[state] != jjround) { jjstateSet[jjnewStateCnt++] = state; jjrounds[state] = jjround; - } -} -private void jjAddStates(int start, int end) -{ - do { + } + } + + private void jjAddStates(int start, int end) { + do { jjstateSet[jjnewStateCnt++] = jjnextStates[start]; - } while (start++ != end); -} -private void jjCheckNAddTwoStates(int state1, int state2) -{ - jjCheckNAdd(state1); - jjCheckNAdd(state2); -} + } while (start++ != end); + } -private void jjCheckNAddStates(int start, int end) -{ - do { + private void jjCheckNAddTwoStates(int state1, int state2) { + jjCheckNAdd(state1); + jjCheckNAdd(state2); + } + + private void jjCheckNAddStates(int start, int end) { + do { jjCheckNAdd(jjnextStates[start]); - } while (start++ != end); -} + } while (start++ != end); + } } Index: serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypeMap.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypeMap.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypeMap.java (working copy) @@ -18,38 +18,44 @@ package org.apache.hadoop.hive.serde2.dynamic_type; -import org.apache.thrift.TException; -import org.apache.thrift.TApplicationException; -import org.apache.thrift.protocol.*; -import org.apache.thrift.server.*; -import org.apache.thrift.transport.*; -import java.util.*; -import java.io.*; -import org.apache.hadoop.hive.serde2.*; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +import org.apache.hadoop.hive.serde2.SerDeException; import org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; - -import java.lang.reflect.*; +import org.apache.thrift.TException; +import org.apache.thrift.protocol.TMap; +import org.apache.thrift.protocol.TProtocol; import org.apache.thrift.protocol.TType; public class DynamicSerDeTypeMap extends DynamicSerDeTypeBase { - public boolean isPrimitive() { return false; } - public boolean isMap() { return true;} + @Override + public boolean isPrimitive() { + return false; + } + @Override + public boolean isMap() { + return true; + } + // production is: Map private final byte FD_KEYTYPE = 0; private final byte FD_VALUETYPE = 1; // returns Map + @Override public Class getRealType() { try { - Class c = this.getKeyType().getRealType(); - Class c2 = this.getValueType().getRealType(); + Class c = getKeyType().getRealType(); + Class c2 = getValueType().getRealType(); Object o = c.newInstance(); Object o2 = c2.newInstance(); - Map l = Collections.singletonMap(o,o2); + Map l = Collections.singletonMap(o, o2); return l.getClass(); } catch (Exception e) { e.printStackTrace(); @@ -62,25 +68,29 @@ } public DynamicSerDeTypeMap(thrift_grammar p, int i) { - super(p,i); + super(p, i); } public DynamicSerDeTypeBase getKeyType() { - return (DynamicSerDeTypeBase)((DynamicSerDeFieldType)this.jjtGetChild(FD_KEYTYPE)).getMyType(); + return ((DynamicSerDeFieldType) jjtGetChild(FD_KEYTYPE)).getMyType(); } public DynamicSerDeTypeBase getValueType() { - return (DynamicSerDeTypeBase)((DynamicSerDeFieldType)this.jjtGetChild(FD_VALUETYPE)).getMyType(); + return ((DynamicSerDeFieldType) jjtGetChild(FD_VALUETYPE)).getMyType(); } + @Override public String toString() { - return "map<" + this.getKeyType().toString() + "," + this.getValueType().toString() + ">"; + return "map<" + getKeyType().toString() + "," + getValueType().toString() + + ">"; } - public Map deserialize(Object reuse, TProtocol iprot) throws SerDeException, TException, IllegalAccessException { + @Override + public Map deserialize(Object reuse, TProtocol iprot) + throws SerDeException, TException, IllegalAccessException { HashMap deserializeReuse; if (reuse != null) { - deserializeReuse = (HashMap)reuse; + deserializeReuse = (HashMap) reuse; deserializeReuse.clear(); } else { deserializeReuse = new HashMap(); @@ -91,57 +101,58 @@ } // themap might be reused by the Protocol. int mapSize = themap.size; - for(int i = 0; i < mapSize; i++) { - Object key = this.getKeyType().deserialize(null, iprot); - Object value = this.getValueType().deserialize(null, iprot); - deserializeReuse.put(key,value); + for (int i = 0; i < mapSize; i++) { + Object key = getKeyType().deserialize(null, iprot); + Object value = getValueType().deserialize(null, iprot); + deserializeReuse.put(key, value); } - // in theory, the below call isn't needed in non thrift_mode, but let's not get too crazy + // in theory, the below call isn't needed in non thrift_mode, but let's not + // get too crazy iprot.readMapEnd(); return deserializeReuse; } TMap serializeMap = null; + @Override public void serialize(Object o, ObjectInspector oi, TProtocol oprot) - throws TException, SerDeException, NoSuchFieldException, - IllegalAccessException { - DynamicSerDeTypeBase keyType = this.getKeyType(); - DynamicSerDeTypeBase valueType = this.getValueType(); + throws TException, SerDeException, NoSuchFieldException, + IllegalAccessException { + DynamicSerDeTypeBase keyType = getKeyType(); + DynamicSerDeTypeBase valueType = getValueType(); - org.apache.hadoop.hive.serde2.thrift.WriteNullsProtocol nullProtocol = - (oprot instanceof org.apache.hadoop.hive.serde2.thrift.WriteNullsProtocol) - ? (org.apache.hadoop.hive.serde2.thrift.WriteNullsProtocol)oprot - : null; - - assert(oi.getCategory() == ObjectInspector.Category.MAP); - MapObjectInspector moi = (MapObjectInspector)oi; + org.apache.hadoop.hive.serde2.thrift.WriteNullsProtocol nullProtocol = (oprot instanceof org.apache.hadoop.hive.serde2.thrift.WriteNullsProtocol) ? (org.apache.hadoop.hive.serde2.thrift.WriteNullsProtocol) oprot + : null; + + assert (oi.getCategory() == ObjectInspector.Category.MAP); + MapObjectInspector moi = (MapObjectInspector) oi; ObjectInspector koi = moi.getMapKeyObjectInspector(); ObjectInspector voi = moi.getMapValueObjectInspector(); - Map map = moi.getMap(o); + Map map = moi.getMap(o); serializeMap = new TMap(keyType.getType(), valueType.getType(), map.size()); oprot.writeMapBegin(serializeMap); - - for(Iterator i = map.entrySet().iterator(); i.hasNext(); ) { - Map.Entry it = (Map.Entry)i.next(); + + for (Object element : map.entrySet()) { + Map.Entry it = (Map.Entry) element; Object key = it.getKey(); Object value = it.getValue(); keyType.serialize(key, koi, oprot); if (value == null) { - assert(nullProtocol != null); + assert (nullProtocol != null); nullProtocol.writeNull(); } else { valueType.serialize(value, voi, oprot); } } - // in theory, the below call isn't needed in non thrift_mode, but let's not get too crazy - oprot.writeMapEnd(); + // in theory, the below call isn't needed in non thrift_mode, but let's not + // get too crazy + oprot.writeMapEnd(); } + @Override public byte getType() { return TType.MAP; } }; - Index: serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeStructBase.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeStructBase.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeStructBase.java (working copy) @@ -18,47 +18,56 @@ package org.apache.hadoop.hive.serde2.dynamic_type; -import org.apache.thrift.TException; -import org.apache.thrift.protocol.*; -import java.io.*; +import java.io.Serializable; import java.util.List; -import org.apache.hadoop.hive.serde2.*; +import org.apache.hadoop.hive.serde2.SerDeException; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.thrift.TException; +import org.apache.thrift.protocol.TProtocol; +import org.apache.thrift.protocol.TStruct; -abstract public class DynamicSerDeStructBase extends DynamicSerDeTypeBase implements Serializable { +abstract public class DynamicSerDeStructBase extends DynamicSerDeTypeBase + implements Serializable { DynamicSerDeFieldList fieldList; public DynamicSerDeStructBase(int i) { super(i); } + public DynamicSerDeStructBase(thrift_grammar p, int i) { - super(p,i); + super(p, i); } abstract public DynamicSerDeFieldList getFieldList(); + @Override public void initialize() { fieldList = getFieldList(); fieldList.initialize(); } - public boolean isPrimitive() { return false; } + @Override + public boolean isPrimitive() { + return false; + } - + @Override public Class getRealType() { return List.class; } - public Object deserialize(Object reuse, TProtocol iprot) throws SerDeException, TException, IllegalAccessException { - if(thrift_mode) { + @Override + public Object deserialize(Object reuse, TProtocol iprot) + throws SerDeException, TException, IllegalAccessException { + if (thrift_mode) { iprot.readStructBegin(); } Object o = fieldList.deserialize(reuse, iprot); - if(thrift_mode) { + if (thrift_mode) { iprot.readStructEnd(); } return o; @@ -66,21 +75,27 @@ /** * serialize - * - * The way to serialize a Thrift "table" which in thrift land is really a function and thus this class's name. - * - * @param o - this list should be in the order of the function's params for now. If we wanted to remove this requirement, - * we'd need to make it a List> with the String being the field name. - * + * + * The way to serialize a Thrift "table" which in thrift land is really a + * function and thus this class's name. + * + * @param o + * - this list should be in the order of the function's params for + * now. If we wanted to remove this requirement, we'd need to make it + * a List> with the String being the field name. + * */ - public void serialize(Object o, ObjectInspector oi, TProtocol oprot) throws TException, SerDeException, NoSuchFieldException,IllegalAccessException { - if(thrift_mode) { - oprot.writeStructBegin(new TStruct(this.name)); + @Override + public void serialize(Object o, ObjectInspector oi, TProtocol oprot) + throws TException, SerDeException, NoSuchFieldException, + IllegalAccessException { + if (thrift_mode) { + oprot.writeStructBegin(new TStruct(name)); } fieldList.serialize(o, oi, oprot); - if(thrift_mode) { + if (thrift_mode) { oprot.writeStructEnd(); } } Index: serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeFieldType.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeFieldType.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeFieldType.java (working copy) @@ -24,17 +24,21 @@ // production: this.name | BaseType() | MapType() | SetType() | ListType() private final int FD_FIELD_TYPE = 0; + public DynamicSerDeFieldType(int i) { super(i); } + public DynamicSerDeFieldType(thrift_grammar p, int i) { - super(p,i); + super(p, i); } protected DynamicSerDeTypeBase getMyType() { - // bugbug, need to deal with a named type here - i.e., look it up and proxy to it - // should raise an exception if this is a typedef since won't be any children + // bugbug, need to deal with a named type here - i.e., look it up and proxy + // to it + // should raise an exception if this is a typedef since won't be any + // children // and thus we can quickly find this comment and limitation. - return (DynamicSerDeTypeBase)this.jjtGetChild(FD_FIELD_TYPE); + return (DynamicSerDeTypeBase) jjtGetChild(FD_FIELD_TYPE); } } Index: serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/Token.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/Token.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/Token.java (working copy) @@ -8,16 +8,16 @@ public class Token { /** - * An integer that describes the kind of this token. This numbering - * system is determined by JavaCCParser, and a table of these numbers is - * stored in the file ...Constants.java. + * An integer that describes the kind of this token. This numbering system is + * determined by JavaCCParser, and a table of these numbers is stored in the + * file ...Constants.java. */ public int kind; /** - * beginLine and beginColumn describe the position of the first character - * of this token; endLine and endColumn describe the position of the - * last character of this token. + * beginLine and beginColumn describe the position of the first character of + * this token; endLine and endColumn describe the position of the last + * character of this token. */ public int beginLine, beginColumn, endLine, endColumn; @@ -27,55 +27,52 @@ public String image; /** - * A reference to the next regular (non-special) token from the input - * stream. If this is the last token from the input stream, or if the - * token manager has not read tokens beyond this one, this field is - * set to null. This is true only if this token is also a regular - * token. Otherwise, see below for a description of the contents of - * this field. + * A reference to the next regular (non-special) token from the input stream. + * If this is the last token from the input stream, or if the token manager + * has not read tokens beyond this one, this field is set to null. This is + * true only if this token is also a regular token. Otherwise, see below for a + * description of the contents of this field. */ public Token next; /** - * This field is used to access special tokens that occur prior to this - * token, but after the immediately preceding regular (non-special) token. - * If there are no such special tokens, this field is set to null. - * When there are more than one such special token, this field refers - * to the last of these special tokens, which in turn refers to the next - * previous special token through its specialToken field, and so on - * until the first special token (whose specialToken field is null). - * The next fields of special tokens refer to other special tokens that - * immediately follow it (without an intervening regular token). If there - * is no such token, this field is null. + * This field is used to access special tokens that occur prior to this token, + * but after the immediately preceding regular (non-special) token. If there + * are no such special tokens, this field is set to null. When there are more + * than one such special token, this field refers to the last of these special + * tokens, which in turn refers to the next previous special token through its + * specialToken field, and so on until the first special token (whose + * specialToken field is null). The next fields of special tokens refer to + * other special tokens that immediately follow it (without an intervening + * regular token). If there is no such token, this field is null. */ public Token specialToken; /** * Returns the image. */ - public String toString() - { - return image; + @Override + public String toString() { + return image; } /** - * Returns a new Token object, by default. However, if you want, you - * can create and return subclass objects based on the value of ofKind. - * Simply add the cases to the switch for all those special cases. - * For example, if you have a subclass of Token called IDToken that - * you want to create if ofKind is ID, simlpy add something like : - * - * case MyParserConstants.ID : return new IDToken(); - * - * to the following switch statement. Then you can cast matchedToken - * variable to the appropriate type and use it in your lexical actions. + * Returns a new Token object, by default. However, if you want, you can + * create and return subclass objects based on the value of ofKind. Simply add + * the cases to the switch for all those special cases. For example, if you + * have a subclass of Token called IDToken that you want to create if ofKind + * is ID, simlpy add something like : + * + * case MyParserConstants.ID : return new IDToken(); + * + * to the following switch statement. Then you can cast matchedToken variable + * to the appropriate type and use it in your lexical actions. */ - public static final Token newToken(int ofKind) - { - switch(ofKind) - { - default : return new Token(); - } + public static final Token newToken(int ofKind) { + switch (ofKind) { + default: + return new Token(); + } } } Index: serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/thrift_grammarTreeConstants.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/thrift_grammarTreeConstants.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/thrift_grammarTreeConstants.java (working copy) @@ -1,8 +1,7 @@ /* Generated By:JavaCC: Do not edit this line. thrift_grammarTreeConstants.java Version 4.1 */ package org.apache.hadoop.hive.serde2.dynamic_type; -public interface thrift_grammarTreeConstants -{ +public interface thrift_grammarTreeConstants { public int JJTSTART = 0; public int JJTHEADERLIST = 1; public int JJTHEADER = 2; @@ -51,55 +50,18 @@ public int JJTTYPESET = 45; public int JJTTYPELIST = 46; - - public String[] jjtNodeName = { - "Start", - "HeaderList", - "Header", - "Namespace", - "Include", - "Definition", - "TypeDefinition", - "Typedef", - "CommaOrSemicolon", - "Enum", - "EnumDefList", - "EnumDef", - "Senum", - "SenumDefList", - "SenumDef", - "Const", - "ConstValue", - "ConstList", - "ConstListContents", - "ConstMap", - "ConstMapContents", - "Struct", - "Xception", - "Service", - "FlagArgs", - "UnflagArgs", - "Extends", - "Function", - "Async", - "Throws", - "FieldList", - "Field", - "FieldRequiredness", - "FieldValue", - "DefinitionType", - "FunctionType", - "FieldType", - "TypeString", - "TypeByte", - "Typei16", - "Typei32", - "Typei64", - "TypeDouble", - "TypeBool", - "TypeMap", - "TypeSet", - "TypeList", - }; + public String[] jjtNodeName = { "Start", "HeaderList", "Header", "Namespace", + "Include", "Definition", "TypeDefinition", "Typedef", "CommaOrSemicolon", + "Enum", "EnumDefList", "EnumDef", "Senum", "SenumDefList", "SenumDef", + "Const", "ConstValue", "ConstList", "ConstListContents", "ConstMap", + "ConstMapContents", "Struct", "Xception", "Service", "FlagArgs", + "UnflagArgs", "Extends", "Function", "Async", "Throws", "FieldList", + "Field", "FieldRequiredness", "FieldValue", "DefinitionType", + "FunctionType", "FieldType", "TypeString", "TypeByte", "Typei16", + "Typei32", "Typei64", "TypeDouble", "TypeBool", "TypeMap", "TypeSet", + "TypeList", }; } -/* JavaCC - OriginalChecksum=7edd3e61472739e9fede55c18a336638 (do not edit this line) */ +/* + * JavaCC - OriginalChecksum=7edd3e61472739e9fede55c18a336638 (do not edit this + * line) + */ Index: serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeSimpleNode.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeSimpleNode.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeSimpleNode.java (working copy) @@ -20,12 +20,15 @@ public class DynamicSerDeSimpleNode extends SimpleNode { protected static final boolean thrift_mode = true; + public DynamicSerDeSimpleNode(int i) { super(i); } + public DynamicSerDeSimpleNode(thrift_grammar p, int i) { - super(p,i); + super(p, i); } + protected int fieldid; protected String name; } Index: serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypeByte.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypeByte.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypeByte.java (working copy) @@ -18,47 +18,57 @@ package org.apache.hadoop.hive.serde2.dynamic_type; - -import org.apache.thrift.TException; -import org.apache.thrift.protocol.*; -import org.apache.hadoop.hive.serde2.*; +import org.apache.hadoop.hive.serde2.SerDeException; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.ByteObjectInspector; - +import org.apache.thrift.TException; +import org.apache.thrift.protocol.TProtocol; import org.apache.thrift.protocol.TType; public class DynamicSerDeTypeByte extends DynamicSerDeTypeBase { // production is: byte - public DynamicSerDeTypeByte(int i) { super(i); } + public DynamicSerDeTypeByte(thrift_grammar p, int i) { - super(p,i); + super(p, i); } - public String toString() { return "byte"; } + @Override + public String toString() { + return "byte"; + } - public Byte deserialize(TProtocol iprot) throws SerDeException, TException, IllegalAccessException { + public Byte deserialize(TProtocol iprot) throws SerDeException, TException, + IllegalAccessException { byte val = iprot.readByte(); - if (val == 0 && iprot instanceof org.apache.hadoop.hive.serde2.thrift.WriteNullsProtocol && - ((org.apache.hadoop.hive.serde2.thrift.WriteNullsProtocol)iprot).lastPrimitiveWasNull()) { + if (val == 0 + && iprot instanceof org.apache.hadoop.hive.serde2.thrift.WriteNullsProtocol + && ((org.apache.hadoop.hive.serde2.thrift.WriteNullsProtocol) iprot) + .lastPrimitiveWasNull()) { return null; } return Byte.valueOf(val); } - public Object deserialize(Object reuse, TProtocol iprot) throws SerDeException, TException, IllegalAccessException { + + @Override + public Object deserialize(Object reuse, TProtocol iprot) + throws SerDeException, TException, IllegalAccessException { return deserialize(iprot); } - public void serialize(Object o, ObjectInspector oi, TProtocol oprot) throws TException, SerDeException, NoSuchFieldException,IllegalAccessException { + @Override + public void serialize(Object o, ObjectInspector oi, TProtocol oprot) + throws TException, SerDeException, NoSuchFieldException, + IllegalAccessException { ByteObjectInspector poi = (ByteObjectInspector) oi; oprot.writeByte(poi.get(o)); } + @Override public byte getType() { return TType.BYTE; } Index: serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/SimpleNode.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/SimpleNode.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/SimpleNode.java (working copy) @@ -22,10 +22,15 @@ public void jjtClose() { } - - public void jjtSetParent(Node n) { parent = n; } - public Node jjtGetParent() { return parent; } + public void jjtSetParent(Node n) { + parent = n; + } + + public Node jjtGetParent() { + return parent; + } + public void jjtAddChild(Node n, int i) { if (children == null) { children = new Node[i + 1]; @@ -45,28 +50,36 @@ return (children == null) ? 0 : children.length; } - /* You can override these two methods in subclasses of SimpleNode to - customize the way the node appears when the tree is dumped. If - your output uses more than one line you should override - toString(String), otherwise overriding toString() is probably all - you need to do. */ + /* + * You can override these two methods in subclasses of SimpleNode to customize + * the way the node appears when the tree is dumped. If your output uses more + * than one line you should override toString(String), otherwise overriding + * toString() is probably all you need to do. + */ - public String toString() { return thrift_grammarTreeConstants.jjtNodeName[id]; } - public String toString(String prefix) { return prefix + toString(); } + @Override + public String toString() { + return thrift_grammarTreeConstants.jjtNodeName[id]; + } - /* Override this method if you want to customize how the node dumps - out its children. */ + public String toString(String prefix) { + return prefix + toString(); + } + /* + * Override this method if you want to customize how the node dumps out its + * children. + */ + public void dump(String prefix) { System.out.println(toString(prefix)); if (children != null) { for (int i = 0; i < children.length; ++i) { - SimpleNode n = (SimpleNode)children[i]; - if (n != null) { - n.dump(prefix + " "); - } + SimpleNode n = (SimpleNode) children[i]; + if (n != null) { + n.dump(prefix + " "); + } } } } } - Index: serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypeDouble.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypeDouble.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypeDouble.java (working copy) @@ -18,19 +18,11 @@ package org.apache.hadoop.hive.serde2.dynamic_type; -import org.apache.thrift.TException; -import org.apache.thrift.TApplicationException; -import org.apache.thrift.protocol.*; -import org.apache.thrift.server.*; -import org.apache.thrift.transport.*; -import java.util.*; -import java.io.*; -import org.apache.hadoop.hive.serde2.*; +import org.apache.hadoop.hive.serde2.SerDeException; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.DoubleObjectInspector; - -import java.lang.reflect.*; +import org.apache.thrift.TException; +import org.apache.thrift.protocol.TProtocol; import org.apache.thrift.protocol.TType; public class DynamicSerDeTypeDouble extends DynamicSerDeTypeBase { @@ -40,30 +32,48 @@ public DynamicSerDeTypeDouble(int i) { super(i); } + public DynamicSerDeTypeDouble(thrift_grammar p, int i) { - super(p,i); + super(p, i); } - public String toString() { return "double"; } + @Override + public String toString() { + return "double"; + } - public Object deserialize(Object reuse, TProtocol iprot) throws SerDeException, TException, IllegalAccessException { + @Override + public Object deserialize(Object reuse, TProtocol iprot) + throws SerDeException, TException, IllegalAccessException { double val = iprot.readDouble(); - if (val == 0 && iprot instanceof org.apache.hadoop.hive.serde2.thrift.WriteNullsProtocol && - ((org.apache.hadoop.hive.serde2.thrift.WriteNullsProtocol)iprot).lastPrimitiveWasNull()) { + if (val == 0 + && iprot instanceof org.apache.hadoop.hive.serde2.thrift.WriteNullsProtocol + && ((org.apache.hadoop.hive.serde2.thrift.WriteNullsProtocol) iprot) + .lastPrimitiveWasNull()) { return null; } return Double.valueOf(val); } - public void serialize(Object o, ObjectInspector oi, TProtocol oprot) throws TException, SerDeException, NoSuchFieldException,IllegalAccessException { + @Override + public void serialize(Object o, ObjectInspector oi, TProtocol oprot) + throws TException, SerDeException, NoSuchFieldException, + IllegalAccessException { DoubleObjectInspector poi = (DoubleObjectInspector) oi; oprot.writeDouble(poi.get(o)); } + @Override public byte getType() { return TType.DOUBLE; } - - public Class getRealType() { return java.lang.Double.class; } - public Double getRealTypeInstance() { return Double.valueOf(0); } + + @Override + public Class getRealType() { + return java.lang.Double.class; + } + + public Double getRealTypeInstance() { + return Double.valueOf(0); + } } Index: serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeFieldRequiredness.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeFieldRequiredness.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeFieldRequiredness.java (working copy) @@ -19,14 +19,13 @@ package org.apache.hadoop.hive.serde2.dynamic_type; public class DynamicSerDeFieldRequiredness extends SimpleNode { - public enum RequirednessTypes - { - Required, Skippable, Optional, - }; + public enum RequirednessTypes { + Required, Skippable, Optional, + }; /** - * Is this a required, skippable or optional field. - * Used by DynamicSerDe for optimizations. + * Is this a required, skippable or optional field. Used by DynamicSerDe for + * optimizations. */ protected RequirednessTypes requiredness; Index: serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeFieldList.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeFieldList.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeFieldList.java (working copy) @@ -18,82 +18,85 @@ package org.apache.hadoop.hive.serde2.dynamic_type; -import org.apache.thrift.TException; -import org.apache.thrift.TApplicationException; -import org.apache.thrift.protocol.*; -import org.apache.thrift.server.*; -import org.apache.thrift.transport.*; -import java.util.*; -import java.io.*; -import org.apache.hadoop.hive.serde2.*; +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.hadoop.hive.serde2.SerDeException; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.StructField; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; +import org.apache.thrift.TException; +import org.apache.thrift.protocol.TField; +import org.apache.thrift.protocol.TProtocol; +import org.apache.thrift.protocol.TProtocolUtil; +import org.apache.thrift.protocol.TType; -import java.lang.reflect.*; -import org.apache.thrift.protocol.TType.*; +public class DynamicSerDeFieldList extends DynamicSerDeSimpleNode implements + Serializable { -public class DynamicSerDeFieldList extends DynamicSerDeSimpleNode implements Serializable { + // private void writeObject(ObjectOutputStream out) throws IOException { + // out.writeObject(types_by_column_name); + // out.writeObject(ordered_types); + // } - // private void writeObject(ObjectOutputStream out) throws IOException { - // out.writeObject(types_by_column_name); - // out.writeObject(ordered_types); - // } - // production: Field()* // mapping of the fieldid to the field - private Map types_by_id = null; - private Map types_by_column_name = null; + private Map types_by_id = null; + private Map types_by_column_name = null; private DynamicSerDeTypeBase ordered_types[] = null; - private Map ordered_column_id_by_name = null; + private Map ordered_column_id_by_name = null; public DynamicSerDeFieldList(int i) { super(i); } public DynamicSerDeFieldList(thrift_grammar p, int i) { - super(p,i); + super(p, i); } private DynamicSerDeField getField(int i) { - return (DynamicSerDeField)this.jjtGetChild(i); + return (DynamicSerDeField) jjtGetChild(i); } - final public DynamicSerDeField [] getChildren() { - int size = this.jjtGetNumChildren(); - DynamicSerDeField result [] = new DynamicSerDeField[size]; - for(int i = 0; i < size; i++) { - result[i] = (DynamicSerDeField)this.jjtGetChild(i); + final public DynamicSerDeField[] getChildren() { + int size = jjtGetNumChildren(); + DynamicSerDeField result[] = new DynamicSerDeField[size]; + for (int i = 0; i < size; i++) { + result[i] = (DynamicSerDeField) jjtGetChild(i); } return result; } private int getNumFields() { - return this.jjtGetNumChildren(); + return jjtGetNumChildren(); } public void initialize() { - if(types_by_id == null) { + if (types_by_id == null) { // multiple means of lookup - types_by_id = new HashMap (); - types_by_column_name = new HashMap (); - ordered_types = new DynamicSerDeTypeBase[this.jjtGetNumChildren()]; + types_by_id = new HashMap(); + types_by_column_name = new HashMap(); + ordered_types = new DynamicSerDeTypeBase[jjtGetNumChildren()]; ordered_column_id_by_name = new HashMap(); // put them in and also roll them up while we're at it // a Field contains a FieldType which in turn contains a type - for(int i = 0 ; i < this.jjtGetNumChildren(); i++) { - DynamicSerDeField mt = this.getField(i); + for (int i = 0; i < jjtGetNumChildren(); i++) { + DynamicSerDeField mt = getField(i); DynamicSerDeTypeBase type = mt.getFieldType().getMyType(); - // types get initialized in case they need to setup any + // types get initialized in case they need to setup any // internal data structures - e.g., DynamicSerDeStructBase type.initialize(); type.fieldid = mt.fieldid; type.name = mt.name; - types_by_id.put(Integer.valueOf(mt.fieldid) , type); + types_by_id.put(Integer.valueOf(mt.fieldid), type); types_by_column_name.put(mt.name, type); ordered_types[i] = type; ordered_column_id_by_name.put(mt.name, i); @@ -110,27 +113,28 @@ } /** - * Indicates whether fields can be out of order or missing. i.e., is it really real - * thrift serialization. - * This is used by dynamicserde to do some optimizations if it knows all the fields exist - * and are required and are serialized in order. - * For now, those optimizations are only done for DynamicSerDe serialized data so always - * set to false for now. + * Indicates whether fields can be out of order or missing. i.e., is it really + * real thrift serialization. This is used by dynamicserde to do some + * optimizations if it knows all the fields exist and are required and are + * serialized in order. For now, those optimizations are only done for + * DynamicSerDe serialized data so always set to false for now. */ protected boolean isRealThrift = false; - protected boolean[] fieldsPresent; - public Object deserialize(Object reuse, TProtocol iprot) throws SerDeException, TException, IllegalAccessException { + protected boolean[] fieldsPresent; + + public Object deserialize(Object reuse, TProtocol iprot) + throws SerDeException, TException, IllegalAccessException { ArrayList struct = null; if (reuse == null) { - struct = new ArrayList(this.getNumFields()); - for(int i=0; i(getNumFields()); + for (DynamicSerDeTypeBase orderedType : ordered_types) { struct.add(null); } } else { struct = (ArrayList) reuse; - assert(struct.size() == ordered_types.length); + assert (struct.size() == ordered_types.length); } boolean fastSkips = iprot instanceof org.apache.hadoop.hive.serde2.thrift.SkippableTProtocol; @@ -144,17 +148,19 @@ Arrays.fill(fieldsPresent, false); // Read the fields. - for(int i = 0; i < this.getNumFields(); i++) { + for (int i = 0; i < getNumFields(); i++) { DynamicSerDeTypeBase mt = null; TField field = null; - - if(!isRealThrift && this.getField(i).isSkippable()) { - // PRE - all the fields are required and serialized in order - is !isRealThrift - mt = this.ordered_types[i]; - if(fastSkips) { - ((org.apache.hadoop.hive.serde2.thrift.SkippableTProtocol)iprot).skip(mt.getType()); + + if (!isRealThrift && getField(i).isSkippable()) { + // PRE - all the fields are required and serialized in order - is + // !isRealThrift + mt = ordered_types[i]; + if (fastSkips) { + ((org.apache.hadoop.hive.serde2.thrift.SkippableTProtocol) iprot) + .skip(mt.getType()); } else { - TProtocolUtil.skip(iprot,mt.getType()); + TProtocolUtil.skip(iprot, mt.getType()); } struct.set(i, null); continue; @@ -162,104 +168,118 @@ if (thrift_mode) { field = iprot.readFieldBegin(); - if(field.type >= 0) { - if(field.type == TType.STOP) { + if (field.type >= 0) { + if (field.type == TType.STOP) { stopSeen = true; break; } - mt = this.getFieldByFieldId(field.id); - if(mt == null) { - System.err.println("ERROR for fieldid: " + field.id + " system has no knowledge of this field which is of type : " + field.type); - TProtocolUtil.skip(iprot,field.type); + mt = getFieldByFieldId(field.id); + if (mt == null) { + System.err.println("ERROR for fieldid: " + field.id + + " system has no knowledge of this field which is of type : " + + field.type); + TProtocolUtil.skip(iprot, field.type); continue; } } } - // field.type < 0 means that this is a faked Thrift field, e.g., TControlSeparatedProtocol, which does not - // serialize the field id in the stream. As a result, the only way to get the field id is to fall back to + // field.type < 0 means that this is a faked Thrift field, e.g., + // TControlSeparatedProtocol, which does not + // serialize the field id in the stream. As a result, the only way to get + // the field id is to fall back to // the position "i". - // The intention of this hack (field.type < 0) is to make TControlSeparatedProtocol a real Thrift prototype, - // but there are a lot additional work to do to fulfill that, and that protocol inherently does not support + // The intention of this hack (field.type < 0) is to make + // TControlSeparatedProtocol a real Thrift prototype, + // but there are a lot additional work to do to fulfill that, and that + // protocol inherently does not support // versioning (adding/deleting fields). int orderedId = -1; if (!thrift_mode || field.type < 0) { - mt = this.ordered_types[i]; - // We don't need to lookup order_column_id_by_name because we know it must be "i". + mt = ordered_types[i]; + // We don't need to lookup order_column_id_by_name because we know it + // must be "i". orderedId = i; } else { // Set the correct position - orderedId = ordered_column_id_by_name.get(mt.name); + orderedId = ordered_column_id_by_name.get(mt.name); } struct.set(orderedId, mt.deserialize(struct.get(orderedId), iprot)); - if(thrift_mode) { + if (thrift_mode) { iprot.readFieldEnd(); } fieldsPresent[orderedId] = true; } - - for(int i = 0; i < ordered_types.length; i++) { + + for (int i = 0; i < ordered_types.length; i++) { if (!fieldsPresent[i]) { struct.set(i, null); } } - - if(thrift_mode && !stopSeen) { - // strip off the STOP marker, which may be left if all the fields were in the serialization + + if (thrift_mode && !stopSeen) { + // strip off the STOP marker, which may be left if all the fields were in + // the serialization iprot.readFieldBegin(); } return struct; } - TField field = new TField(); - public void serialize(Object o, ObjectInspector oi, TProtocol oprot) throws TException, SerDeException, NoSuchFieldException,IllegalAccessException { - // Assuming the ObjectInspector represents exactly the same type as this struct. + + public void serialize(Object o, ObjectInspector oi, TProtocol oprot) + throws TException, SerDeException, NoSuchFieldException, + IllegalAccessException { + // Assuming the ObjectInspector represents exactly the same type as this + // struct. // This assumption should be checked during query compile time. - assert(oi instanceof StructObjectInspector); + assert (oi instanceof StructObjectInspector); StructObjectInspector soi = (StructObjectInspector) oi; boolean writeNulls = oprot instanceof org.apache.hadoop.hive.serde2.thrift.WriteNullsProtocol; - + // For every field List fields = soi.getAllStructFieldRefs(); if (fields.size() != ordered_types.length) { - throw new SerDeException("Trying to serialize " + fields.size() - + " fields into a struct with " + ordered_types.length - + " object=" + o + " objectinspector=" + oi.getTypeName()); + throw new SerDeException("Trying to serialize " + fields.size() + + " fields into a struct with " + ordered_types.length + " object=" + + o + " objectinspector=" + oi.getTypeName()); } - for (int i=0; i mk) { popNode(); } - mk = ((Integer)marks.remove(marks.size()-1)).intValue(); + mk = ((Integer) marks.remove(marks.size() - 1)).intValue(); } - public void openNodeScope(Node n) { marks.add(new Integer(mk)); mk = sp; n.jjtOpen(); } - - /* A definite node is constructed from a specified number of - children. That number of nodes are popped from the stack and - made the children of the definite node. Then the definite node - is pushed on to the stack. */ + /* + * A definite node is constructed from a specified number of children. That + * number of nodes are popped from the stack and made the children of the + * definite node. Then the definite node is pushed on to the stack. + */ public void closeNodeScope(Node n, int num) { - mk = ((Integer)marks.remove(marks.size()-1)).intValue(); + mk = ((Integer) marks.remove(marks.size() - 1)).intValue(); while (num-- > 0) { Node c = popNode(); c.jjtSetParent(n); @@ -96,16 +101,16 @@ node_created = true; } - - /* A conditional node is constructed if its condition is true. All - the nodes that have been pushed since the node was opened are - made children of the conditional node, which is then pushed - on to the stack. If the condition is false the node is not - constructed and they are left on the stack. */ + /* + * A conditional node is constructed if its condition is true. All the nodes + * that have been pushed since the node was opened are made children of the + * conditional node, which is then pushed on to the stack. If the condition is + * false the node is not constructed and they are left on the stack. + */ public void closeNodeScope(Node n, boolean condition) { if (condition) { int a = nodeArity(); - mk = ((Integer)marks.remove(marks.size()-1)).intValue(); + mk = ((Integer) marks.remove(marks.size() - 1)).intValue(); while (a-- > 0) { Node c = popNode(); c.jjtSetParent(n); @@ -115,9 +120,12 @@ pushNode(n); node_created = true; } else { - mk = ((Integer)marks.remove(marks.size()-1)).intValue(); + mk = ((Integer) marks.remove(marks.size() - 1)).intValue(); node_created = false; } } } -/* JavaCC - OriginalChecksum=67039445e12d18e18e63124a33879cd3 (do not edit this line) */ +/* + * JavaCC - OriginalChecksum=67039445e12d18e18e63124a33879cd3 (do not edit this + * line) + */ Index: serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/Node.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/Node.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/Node.java (working copy) @@ -3,30 +3,39 @@ package org.apache.hadoop.hive.serde2.dynamic_type; /* All AST nodes must implement this interface. It provides basic - machinery for constructing the parent and child relationships - between nodes. */ + machinery for constructing the parent and child relationships + between nodes. */ public interface Node { - /** This method is called after the node has been made the current - node. It indicates that child nodes can now be added to it. */ + /** + * This method is called after the node has been made the current node. It + * indicates that child nodes can now be added to it. + */ public void jjtOpen(); - /** This method is called after all the child nodes have been - added. */ + /** + * This method is called after all the child nodes have been added. + */ public void jjtClose(); - /** This pair of methods are used to inform the node of its - parent. */ + /** + * This pair of methods are used to inform the node of its parent. + */ public void jjtSetParent(Node n); + public Node jjtGetParent(); - /** This method tells the node to add its argument to the node's - list of children. */ + /** + * This method tells the node to add its argument to the node's list of + * children. + */ public void jjtAddChild(Node n, int i); - /** This method returns a child node. The children are numbered - from zero, left to right. */ + /** + * This method returns a child node. The children are numbered from zero, left + * to right. + */ public Node jjtGetChild(int i); /** Return the number of children the node has. */ Index: serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/TokenMgrError.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/TokenMgrError.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/TokenMgrError.java (working copy) @@ -1,133 +1,136 @@ /* Generated By:JavaCC: Do not edit this line. TokenMgrError.java Version 3.0 */ package org.apache.hadoop.hive.serde2.dynamic_type; -public class TokenMgrError extends Error -{ - /* - * Ordinals for various reasons why an Error of this type can be thrown. - */ +public class TokenMgrError extends Error { + /* + * Ordinals for various reasons why an Error of this type can be thrown. + */ - /** - * Lexical error occured. - */ - static final int LEXICAL_ERROR = 0; + /** + * Lexical error occured. + */ + static final int LEXICAL_ERROR = 0; - /** - * An attempt wass made to create a second instance of a static token manager. - */ - static final int STATIC_LEXER_ERROR = 1; + /** + * An attempt wass made to create a second instance of a static token manager. + */ + static final int STATIC_LEXER_ERROR = 1; - /** - * Tried to change to an invalid lexical state. - */ - static final int INVALID_LEXICAL_STATE = 2; + /** + * Tried to change to an invalid lexical state. + */ + static final int INVALID_LEXICAL_STATE = 2; - /** - * Detected (and bailed out of) an infinite loop in the token manager. - */ - static final int LOOP_DETECTED = 3; + /** + * Detected (and bailed out of) an infinite loop in the token manager. + */ + static final int LOOP_DETECTED = 3; - /** - * Indicates the reason why the exception is thrown. It will have - * one of the above 4 values. - */ - int errorCode; + /** + * Indicates the reason why the exception is thrown. It will have one of the + * above 4 values. + */ + int errorCode; - /** - * Replaces unprintable characters by their espaced (or unicode escaped) - * equivalents in the given string - */ - protected static final String addEscapes(String str) { - StringBuffer retval = new StringBuffer(); - char ch; - for (int i = 0; i < str.length(); i++) { - switch (str.charAt(i)) - { - case 0 : - continue; - case '\b': - retval.append("\\b"); - continue; - case '\t': - retval.append("\\t"); - continue; - case '\n': - retval.append("\\n"); - continue; - case '\f': - retval.append("\\f"); - continue; - case '\r': - retval.append("\\r"); - continue; - case '\"': - retval.append("\\\""); - continue; - case '\'': - retval.append("\\\'"); - continue; - case '\\': - retval.append("\\\\"); - continue; - default: - if ((ch = str.charAt(i)) < 0x20 || ch > 0x7e) { - String s = "0000" + Integer.toString(ch, 16); - retval.append("\\u" + s.substring(s.length() - 4, s.length())); - } else { - retval.append(ch); - } - continue; + /** + * Replaces unprintable characters by their espaced (or unicode escaped) + * equivalents in the given string + */ + protected static final String addEscapes(String str) { + StringBuffer retval = new StringBuffer(); + char ch; + for (int i = 0; i < str.length(); i++) { + switch (str.charAt(i)) { + case 0: + continue; + case '\b': + retval.append("\\b"); + continue; + case '\t': + retval.append("\\t"); + continue; + case '\n': + retval.append("\\n"); + continue; + case '\f': + retval.append("\\f"); + continue; + case '\r': + retval.append("\\r"); + continue; + case '\"': + retval.append("\\\""); + continue; + case '\'': + retval.append("\\\'"); + continue; + case '\\': + retval.append("\\\\"); + continue; + default: + if ((ch = str.charAt(i)) < 0x20 || ch > 0x7e) { + String s = "0000" + Integer.toString(ch, 16); + retval.append("\\u" + s.substring(s.length() - 4, s.length())); + } else { + retval.append(ch); } + continue; } - return retval.toString(); - } + } + return retval.toString(); + } - /** - * Returns a detailed message for the Error when it is thrown by the - * token manager to indicate a lexical error. - * Parameters : - * EOFSeen : indicates if EOF caused the lexicl error - * curLexState : lexical state in which this error occured - * errorLine : line number when the error occured - * errorColumn : column number when the error occured - * errorAfter : prefix that was seen before this error occured - * curchar : the offending character - * Note: You can customize the lexical error message by modifying this method. - */ - protected static String LexicalError(boolean EOFSeen, int lexState, int errorLine, int errorColumn, String errorAfter, char curChar) { - return("Lexical error at line " + - errorLine + ", column " + - errorColumn + ". Encountered: " + - (EOFSeen ? " " : ("\"" + addEscapes(String.valueOf(curChar)) + "\"") + " (" + (int)curChar + "), ") + - "after : \"" + addEscapes(errorAfter) + "\""); - } + /** + * Returns a detailed message for the Error when it is thrown by the token + * manager to indicate a lexical error. Parameters : EOFSeen : indicates if + * EOF caused the lexicl error curLexState : lexical state in which this error + * occured errorLine : line number when the error occured errorColumn : column + * number when the error occured errorAfter : prefix that was seen before this + * error occured curchar : the offending character Note: You can customize the + * lexical error message by modifying this method. + */ + protected static String LexicalError(boolean EOFSeen, int lexState, + int errorLine, int errorColumn, String errorAfter, char curChar) { + return ("Lexical error at line " + + errorLine + + ", column " + + errorColumn + + ". Encountered: " + + (EOFSeen ? " " + : ("\"" + addEscapes(String.valueOf(curChar)) + "\"") + " (" + + (int) curChar + "), ") + "after : \"" + + addEscapes(errorAfter) + "\""); + } - /** - * You can also modify the body of this method to customize your error messages. - * For example, cases like LOOP_DETECTED and INVALID_LEXICAL_STATE are not - * of end-users concern, so you can return something like : - * - * "Internal Error : Please file a bug report .... " - * - * from this method for such cases in the release version of your parser. - */ - public String getMessage() { - return super.getMessage(); - } + /** + * You can also modify the body of this method to customize your error + * messages. For example, cases like LOOP_DETECTED and INVALID_LEXICAL_STATE + * are not of end-users concern, so you can return something like : + * + * "Internal Error : Please file a bug report .... " + * + * from this method for such cases in the release version of your parser. + */ + @Override + public String getMessage() { + return super.getMessage(); + } - /* - * Constructors of various flavors follow. - */ + /* + * Constructors of various flavors follow. + */ - public TokenMgrError() { - } + public TokenMgrError() { + } - public TokenMgrError(String message, int reason) { - super(message); - errorCode = reason; - } + public TokenMgrError(String message, int reason) { + super(message); + errorCode = reason; + } - public TokenMgrError(boolean EOFSeen, int lexState, int errorLine, int errorColumn, String errorAfter, char curChar, int reason) { - this(LexicalError(EOFSeen, lexState, errorLine, errorColumn, errorAfter, curChar), reason); - } + public TokenMgrError(boolean EOFSeen, int lexState, int errorLine, + int errorColumn, String errorAfter, char curChar, int reason) { + this(LexicalError(EOFSeen, lexState, errorLine, errorColumn, errorAfter, + curChar), reason); + } } Index: serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/SimpleCharStream.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/SimpleCharStream.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/SimpleCharStream.java (working copy) @@ -6,8 +6,7 @@ * contain only ASCII characters (without unicode processing). */ -public class SimpleCharStream -{ +public class SimpleCharStream { public static final boolean staticFlag = false; int bufsize; int available; @@ -29,219 +28,211 @@ protected int inBuf = 0; protected int tabSize = 8; - protected void setTabSize(int i) { tabSize = i; } - protected int getTabSize(int i) { return tabSize; } + protected void setTabSize(int i) { + tabSize = i; + } + protected int getTabSize(int i) { + return tabSize; + } - protected void ExpandBuff(boolean wrapAround) - { - char[] newbuffer = new char[bufsize + 2048]; - int newbufline[] = new int[bufsize + 2048]; - int newbufcolumn[] = new int[bufsize + 2048]; + protected void ExpandBuff(boolean wrapAround) { + char[] newbuffer = new char[bufsize + 2048]; + int newbufline[] = new int[bufsize + 2048]; + int newbufcolumn[] = new int[bufsize + 2048]; - try - { - if (wrapAround) - { - System.arraycopy(buffer, tokenBegin, newbuffer, 0, bufsize - tokenBegin); - System.arraycopy(buffer, 0, newbuffer, - bufsize - tokenBegin, bufpos); - buffer = newbuffer; + try { + if (wrapAround) { + System + .arraycopy(buffer, tokenBegin, newbuffer, 0, bufsize - tokenBegin); + System.arraycopy(buffer, 0, newbuffer, bufsize - tokenBegin, bufpos); + buffer = newbuffer; - System.arraycopy(bufline, tokenBegin, newbufline, 0, bufsize - tokenBegin); - System.arraycopy(bufline, 0, newbufline, bufsize - tokenBegin, bufpos); - bufline = newbufline; + System.arraycopy(bufline, tokenBegin, newbufline, 0, bufsize + - tokenBegin); + System.arraycopy(bufline, 0, newbufline, bufsize - tokenBegin, bufpos); + bufline = newbufline; - System.arraycopy(bufcolumn, tokenBegin, newbufcolumn, 0, bufsize - tokenBegin); - System.arraycopy(bufcolumn, 0, newbufcolumn, bufsize - tokenBegin, bufpos); - bufcolumn = newbufcolumn; + System.arraycopy(bufcolumn, tokenBegin, newbufcolumn, 0, bufsize + - tokenBegin); + System.arraycopy(bufcolumn, 0, newbufcolumn, bufsize - tokenBegin, + bufpos); + bufcolumn = newbufcolumn; - maxNextCharInd = (bufpos += (bufsize - tokenBegin)); - } - else - { - System.arraycopy(buffer, tokenBegin, newbuffer, 0, bufsize - tokenBegin); - buffer = newbuffer; + maxNextCharInd = (bufpos += (bufsize - tokenBegin)); + } else { + System + .arraycopy(buffer, tokenBegin, newbuffer, 0, bufsize - tokenBegin); + buffer = newbuffer; - System.arraycopy(bufline, tokenBegin, newbufline, 0, bufsize - tokenBegin); - bufline = newbufline; + System.arraycopy(bufline, tokenBegin, newbufline, 0, bufsize + - tokenBegin); + bufline = newbufline; - System.arraycopy(bufcolumn, tokenBegin, newbufcolumn, 0, bufsize - tokenBegin); - bufcolumn = newbufcolumn; + System.arraycopy(bufcolumn, tokenBegin, newbufcolumn, 0, bufsize + - tokenBegin); + bufcolumn = newbufcolumn; - maxNextCharInd = (bufpos -= tokenBegin); - } - } - catch (Throwable t) - { - throw new Error(t.getMessage()); - } + maxNextCharInd = (bufpos -= tokenBegin); + } + } catch (Throwable t) { + throw new Error(t.getMessage()); + } - - bufsize += 2048; - available = bufsize; - tokenBegin = 0; + bufsize += 2048; + available = bufsize; + tokenBegin = 0; } - protected void FillBuff() throws java.io.IOException - { - if (maxNextCharInd == available) - { - if (available == bufsize) - { - if (tokenBegin > 2048) - { - bufpos = maxNextCharInd = 0; - available = tokenBegin; - } - else if (tokenBegin < 0) - bufpos = maxNextCharInd = 0; - else - ExpandBuff(false); + protected void FillBuff() throws java.io.IOException { + if (maxNextCharInd == available) { + if (available == bufsize) { + if (tokenBegin > 2048) { + bufpos = maxNextCharInd = 0; + available = tokenBegin; + } else if (tokenBegin < 0) { + bufpos = maxNextCharInd = 0; + } else { + ExpandBuff(false); } - else if (available > tokenBegin) - available = bufsize; - else if ((tokenBegin - available) < 2048) - ExpandBuff(true); - else - available = tokenBegin; - } + } else if (available > tokenBegin) { + available = bufsize; + } else if ((tokenBegin - available) < 2048) { + ExpandBuff(true); + } else { + available = tokenBegin; + } + } - int i; - try { - if ((i = inputStream.read(buffer, maxNextCharInd, - available - maxNextCharInd)) == -1) - { - inputStream.close(); - throw new java.io.IOException(); - } - else - maxNextCharInd += i; - return; - } - catch(java.io.IOException e) { - --bufpos; - backup(0); - if (tokenBegin == -1) - tokenBegin = bufpos; - throw e; - } + int i; + try { + if ((i = inputStream.read(buffer, maxNextCharInd, available + - maxNextCharInd)) == -1) { + inputStream.close(); + throw new java.io.IOException(); + } else { + maxNextCharInd += i; + } + return; + } catch (java.io.IOException e) { + --bufpos; + backup(0); + if (tokenBegin == -1) { + tokenBegin = bufpos; + } + throw e; + } } - public char BeginToken() throws java.io.IOException - { - tokenBegin = -1; - char c = readChar(); - tokenBegin = bufpos; + public char BeginToken() throws java.io.IOException { + tokenBegin = -1; + char c = readChar(); + tokenBegin = bufpos; - return c; + return c; } - protected void UpdateLineColumn(char c) - { - column++; + protected void UpdateLineColumn(char c) { + column++; - if (prevCharIsLF) - { - prevCharIsLF = false; + if (prevCharIsLF) { + prevCharIsLF = false; + line += (column = 1); + } else if (prevCharIsCR) { + prevCharIsCR = false; + if (c == '\n') { + prevCharIsLF = true; + } else { line += (column = 1); - } - else if (prevCharIsCR) - { - prevCharIsCR = false; - if (c == '\n') - { - prevCharIsLF = true; - } - else - line += (column = 1); - } + } + } - switch (c) - { - case '\r' : - prevCharIsCR = true; - break; - case '\n' : - prevCharIsLF = true; - break; - case '\t' : - column--; - column += (tabSize - (column % tabSize)); - break; - default : - break; - } + switch (c) { + case '\r': + prevCharIsCR = true; + break; + case '\n': + prevCharIsLF = true; + break; + case '\t': + column--; + column += (tabSize - (column % tabSize)); + break; + default: + break; + } - bufline[bufpos] = line; - bufcolumn[bufpos] = column; + bufline[bufpos] = line; + bufcolumn[bufpos] = column; } - public char readChar() throws java.io.IOException - { - if (inBuf > 0) - { - --inBuf; + public char readChar() throws java.io.IOException { + if (inBuf > 0) { + --inBuf; - if (++bufpos == bufsize) - bufpos = 0; + if (++bufpos == bufsize) { + bufpos = 0; + } - return buffer[bufpos]; - } + return buffer[bufpos]; + } - if (++bufpos >= maxNextCharInd) - FillBuff(); + if (++bufpos >= maxNextCharInd) { + FillBuff(); + } - char c = buffer[bufpos]; + char c = buffer[bufpos]; - UpdateLineColumn(c); - return (c); + UpdateLineColumn(c); + return (c); } /** - * @deprecated + * @deprecated * @see #getEndColumn */ + @Deprecated public int getColumn() { - return bufcolumn[bufpos]; + return bufcolumn[bufpos]; } /** - * @deprecated + * @deprecated * @see #getEndLine */ + @Deprecated public int getLine() { - return bufline[bufpos]; + return bufline[bufpos]; } public int getEndColumn() { - return bufcolumn[bufpos]; + return bufcolumn[bufpos]; } public int getEndLine() { - return bufline[bufpos]; + return bufline[bufpos]; } public int getBeginColumn() { - return bufcolumn[tokenBegin]; + return bufcolumn[tokenBegin]; } public int getBeginLine() { - return bufline[tokenBegin]; + return bufline[tokenBegin]; } public void backup(int amount) { inBuf += amount; - if ((bufpos -= amount) < 0) - bufpos += bufsize; + if ((bufpos -= amount) < 0) { + bufpos += bufsize; + } } public SimpleCharStream(java.io.Reader dstream, int startline, - int startcolumn, int buffersize) - { + int startcolumn, int buffersize) { inputStream = dstream; line = startline; column = startcolumn - 1; @@ -252,25 +243,21 @@ bufcolumn = new int[buffersize]; } - public SimpleCharStream(java.io.Reader dstream, int startline, - int startcolumn) - { - this(dstream, startline, startcolumn, 4096); + public SimpleCharStream(java.io.Reader dstream, int startline, int startcolumn) { + this(dstream, startline, startcolumn, 4096); } - public SimpleCharStream(java.io.Reader dstream) - { - this(dstream, 1, 1, 4096); + public SimpleCharStream(java.io.Reader dstream) { + this(dstream, 1, 1, 4096); } - public void ReInit(java.io.Reader dstream, int startline, - int startcolumn, int buffersize) - { + + public void ReInit(java.io.Reader dstream, int startline, int startcolumn, + int buffersize) { inputStream = dstream; line = startline; column = startcolumn - 1; - if (buffer == null || buffersize != buffer.length) - { + if (buffer == null || buffersize != buffer.length) { available = bufsize = buffersize; buffer = new char[buffersize]; bufline = new int[buffersize]; @@ -281,159 +268,150 @@ bufpos = -1; } - public void ReInit(java.io.Reader dstream, int startline, - int startcolumn) - { - ReInit(dstream, startline, startcolumn, 4096); + public void ReInit(java.io.Reader dstream, int startline, int startcolumn) { + ReInit(dstream, startline, startcolumn, 4096); } - public void ReInit(java.io.Reader dstream) - { - ReInit(dstream, 1, 1, 4096); + public void ReInit(java.io.Reader dstream) { + ReInit(dstream, 1, 1, 4096); } - public SimpleCharStream(java.io.InputStream dstream, String encoding, int startline, - int startcolumn, int buffersize) throws java.io.UnsupportedEncodingException - { - this(encoding == null ? new java.io.InputStreamReader(dstream) : new java.io.InputStreamReader(dstream, encoding), startline, startcolumn, buffersize); + + public SimpleCharStream(java.io.InputStream dstream, String encoding, + int startline, int startcolumn, int buffersize) + throws java.io.UnsupportedEncodingException { + this(encoding == null ? new java.io.InputStreamReader(dstream) + : new java.io.InputStreamReader(dstream, encoding), startline, + startcolumn, buffersize); } public SimpleCharStream(java.io.InputStream dstream, int startline, - int startcolumn, int buffersize) - { - this(new java.io.InputStreamReader(dstream), startline, startcolumn, buffersize); + int startcolumn, int buffersize) { + this(new java.io.InputStreamReader(dstream), startline, startcolumn, + buffersize); } - public SimpleCharStream(java.io.InputStream dstream, String encoding, int startline, - int startcolumn) throws java.io.UnsupportedEncodingException - { - this(dstream, encoding, startline, startcolumn, 4096); + public SimpleCharStream(java.io.InputStream dstream, String encoding, + int startline, int startcolumn) + throws java.io.UnsupportedEncodingException { + this(dstream, encoding, startline, startcolumn, 4096); } public SimpleCharStream(java.io.InputStream dstream, int startline, - int startcolumn) - { - this(dstream, startline, startcolumn, 4096); + int startcolumn) { + this(dstream, startline, startcolumn, 4096); } - public SimpleCharStream(java.io.InputStream dstream, String encoding) throws java.io.UnsupportedEncodingException - { - this(dstream, encoding, 1, 1, 4096); + public SimpleCharStream(java.io.InputStream dstream, String encoding) + throws java.io.UnsupportedEncodingException { + this(dstream, encoding, 1, 1, 4096); } - public SimpleCharStream(java.io.InputStream dstream) - { - this(dstream, 1, 1, 4096); + public SimpleCharStream(java.io.InputStream dstream) { + this(dstream, 1, 1, 4096); } - public void ReInit(java.io.InputStream dstream, String encoding, int startline, - int startcolumn, int buffersize) throws java.io.UnsupportedEncodingException - { - ReInit(encoding == null ? new java.io.InputStreamReader(dstream) : new java.io.InputStreamReader(dstream, encoding), startline, startcolumn, buffersize); + public void ReInit(java.io.InputStream dstream, String encoding, + int startline, int startcolumn, int buffersize) + throws java.io.UnsupportedEncodingException { + ReInit(encoding == null ? new java.io.InputStreamReader(dstream) + : new java.io.InputStreamReader(dstream, encoding), startline, + startcolumn, buffersize); } public void ReInit(java.io.InputStream dstream, int startline, - int startcolumn, int buffersize) - { - ReInit(new java.io.InputStreamReader(dstream), startline, startcolumn, buffersize); + int startcolumn, int buffersize) { + ReInit(new java.io.InputStreamReader(dstream), startline, startcolumn, + buffersize); } - public void ReInit(java.io.InputStream dstream, String encoding) throws java.io.UnsupportedEncodingException - { - ReInit(dstream, encoding, 1, 1, 4096); + public void ReInit(java.io.InputStream dstream, String encoding) + throws java.io.UnsupportedEncodingException { + ReInit(dstream, encoding, 1, 1, 4096); } - public void ReInit(java.io.InputStream dstream) - { - ReInit(dstream, 1, 1, 4096); + public void ReInit(java.io.InputStream dstream) { + ReInit(dstream, 1, 1, 4096); } - public void ReInit(java.io.InputStream dstream, String encoding, int startline, - int startcolumn) throws java.io.UnsupportedEncodingException - { - ReInit(dstream, encoding, startline, startcolumn, 4096); + + public void ReInit(java.io.InputStream dstream, String encoding, + int startline, int startcolumn) + throws java.io.UnsupportedEncodingException { + ReInit(dstream, encoding, startline, startcolumn, 4096); } - public void ReInit(java.io.InputStream dstream, int startline, - int startcolumn) - { - ReInit(dstream, startline, startcolumn, 4096); + + public void ReInit(java.io.InputStream dstream, int startline, int startcolumn) { + ReInit(dstream, startline, startcolumn, 4096); } - public String GetImage() - { - if (bufpos >= tokenBegin) - return new String(buffer, tokenBegin, bufpos - tokenBegin + 1); - else - return new String(buffer, tokenBegin, bufsize - tokenBegin) + - new String(buffer, 0, bufpos + 1); + + public String GetImage() { + if (bufpos >= tokenBegin) { + return new String(buffer, tokenBegin, bufpos - tokenBegin + 1); + } else { + return new String(buffer, tokenBegin, bufsize - tokenBegin) + + new String(buffer, 0, bufpos + 1); + } } - public char[] GetSuffix(int len) - { - char[] ret = new char[len]; + public char[] GetSuffix(int len) { + char[] ret = new char[len]; - if ((bufpos + 1) >= len) - System.arraycopy(buffer, bufpos - len + 1, ret, 0, len); - else - { - System.arraycopy(buffer, bufsize - (len - bufpos - 1), ret, 0, - len - bufpos - 1); - System.arraycopy(buffer, 0, ret, len - bufpos - 1, bufpos + 1); - } + if ((bufpos + 1) >= len) { + System.arraycopy(buffer, bufpos - len + 1, ret, 0, len); + } else { + System.arraycopy(buffer, bufsize - (len - bufpos - 1), ret, 0, len + - bufpos - 1); + System.arraycopy(buffer, 0, ret, len - bufpos - 1, bufpos + 1); + } - return ret; + return ret; } - public void Done() - { - buffer = null; - bufline = null; - bufcolumn = null; + public void Done() { + buffer = null; + bufline = null; + bufcolumn = null; } /** * Method to adjust line and column numbers for the start of a token. */ - public void adjustBeginLineColumn(int newLine, int newCol) - { - int start = tokenBegin; - int len; + public void adjustBeginLineColumn(int newLine, int newCol) { + int start = tokenBegin; + int len; - if (bufpos >= tokenBegin) - { - len = bufpos - tokenBegin + inBuf + 1; - } - else - { - len = bufsize - tokenBegin + bufpos + 1 + inBuf; - } + if (bufpos >= tokenBegin) { + len = bufpos - tokenBegin + inBuf + 1; + } else { + len = bufsize - tokenBegin + bufpos + 1 + inBuf; + } - int i = 0, j = 0, k = 0; - int nextColDiff = 0, columnDiff = 0; + int i = 0, j = 0, k = 0; + int nextColDiff = 0, columnDiff = 0; - while (i < len && - bufline[j = start % bufsize] == bufline[k = ++start % bufsize]) - { - bufline[j] = newLine; - nextColDiff = columnDiff + bufcolumn[k] - bufcolumn[j]; - bufcolumn[j] = newCol + columnDiff; - columnDiff = nextColDiff; - i++; - } + while (i < len + && bufline[j = start % bufsize] == bufline[k = ++start % bufsize]) { + bufline[j] = newLine; + nextColDiff = columnDiff + bufcolumn[k] - bufcolumn[j]; + bufcolumn[j] = newCol + columnDiff; + columnDiff = nextColDiff; + i++; + } - if (i < len) - { - bufline[j] = newLine++; - bufcolumn[j] = newCol + columnDiff; + if (i < len) { + bufline[j] = newLine++; + bufcolumn[j] = newCol + columnDiff; - while (i++ < len) - { - if (bufline[j = start % bufsize] != bufline[++start % bufsize]) - bufline[j] = newLine++; - else - bufline[j] = newLine; + while (i++ < len) { + if (bufline[j = start % bufsize] != bufline[++start % bufsize]) { + bufline[j] = newLine++; + } else { + bufline[j] = newLine; } - } + } + } - line = bufline[j]; - column = bufcolumn[j]; + line = bufline[j]; + column = bufcolumn[j]; } } Index: serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypeBase.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypeBase.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypeBase.java (working copy) @@ -18,14 +18,15 @@ package org.apache.hadoop.hive.serde2.dynamic_type; -import org.apache.hadoop.hive.serde2.*; +import java.io.Serializable; + +import org.apache.hadoop.hive.serde2.SerDeException; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; - +import org.apache.thrift.TException; import org.apache.thrift.protocol.TProtocol; -import org.apache.thrift.TException; -import java.io.Serializable; -public abstract class DynamicSerDeTypeBase extends DynamicSerDeSimpleNode implements Serializable { +public abstract class DynamicSerDeTypeBase extends DynamicSerDeSimpleNode + implements Serializable { private static final long serialVersionUID = 1L; public DynamicSerDeTypeBase(int i) { @@ -33,11 +34,12 @@ } public DynamicSerDeTypeBase(thrift_grammar p, int i) { - super(p,i); + super(p, i); } public void initialize() { - // for base type, do nothing. Other types, like structs may initialize internal data + // for base type, do nothing. Other types, like structs may initialize + // internal data // structures. } @@ -49,10 +51,14 @@ throw new RuntimeException("Not implemented in base"); } - public abstract Object deserialize(Object reuse, TProtocol iprot) throws SerDeException, TException, IllegalAccessException; + public abstract Object deserialize(Object reuse, TProtocol iprot) + throws SerDeException, TException, IllegalAccessException; - public abstract void serialize(Object o, ObjectInspector oi, TProtocol oprot) throws TException, SerDeException, NoSuchFieldException,IllegalAccessException; + public abstract void serialize(Object o, ObjectInspector oi, TProtocol oprot) + throws TException, SerDeException, NoSuchFieldException, + IllegalAccessException; + @Override public String toString() { return "BAD"; } @@ -61,8 +67,16 @@ return -1; } - public boolean isPrimitive() { return true; } - public boolean isList() { return false; } - public boolean isMap() { return false; } + public boolean isPrimitive() { + return true; + } + public boolean isList() { + return false; + } + + public boolean isMap() { + return false; + } + } Index: serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypeList.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypeList.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypeList.java (working copy) @@ -18,25 +18,35 @@ package org.apache.hadoop.hive.serde2.dynamic_type; -import org.apache.thrift.TException; -import org.apache.thrift.protocol.*; -import java.util.*; +import java.util.ArrayList; +import java.util.List; + import org.apache.hadoop.hive.serde.Constants; -import org.apache.hadoop.hive.serde2.*; +import org.apache.hadoop.hive.serde2.SerDeException; import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; - +import org.apache.thrift.TException; +import org.apache.thrift.protocol.TList; +import org.apache.thrift.protocol.TProtocol; import org.apache.thrift.protocol.TType; public class DynamicSerDeTypeList extends DynamicSerDeTypeBase { - public boolean isPrimitive() { return false; } - public boolean isList() { return true; } + @Override + public boolean isPrimitive() { + return false; + } + @Override + public boolean isList() { + return true; + } + // production is: list static final private int FD_TYPE = 0; + @Override public Class getRealType() { return java.util.ArrayList.class; } @@ -44,20 +54,23 @@ public DynamicSerDeTypeList(int i) { super(i); } + public DynamicSerDeTypeList(thrift_grammar p, int i) { - super(p,i); + super(p, i); } public DynamicSerDeTypeBase getElementType() { - return (DynamicSerDeTypeBase)((DynamicSerDeFieldType)this.jjtGetChild(FD_TYPE)).getMyType(); + return ((DynamicSerDeFieldType) jjtGetChild(FD_TYPE)).getMyType(); } + @Override public String toString() { - return Constants.LIST_TYPE_NAME + "<" + this.getElementType().toString() + ">"; + return Constants.LIST_TYPE_NAME + "<" + getElementType().toString() + ">"; } @Override - public ArrayList deserialize(Object reuse, TProtocol iprot) throws SerDeException, TException, IllegalAccessException { + public ArrayList deserialize(Object reuse, TProtocol iprot) + throws SerDeException, TException, IllegalAccessException { TList thelist = iprot.readListBegin(); if (thelist == null) { return null; @@ -65,66 +78,70 @@ ArrayList deserializeReuse; if (reuse != null) { - deserializeReuse = (ArrayList)reuse; + deserializeReuse = (ArrayList) reuse; // Trim to the size needed while (deserializeReuse.size() > thelist.size) { - deserializeReuse.remove(deserializeReuse.size()-1); + deserializeReuse.remove(deserializeReuse.size() - 1); } } else { deserializeReuse = new ArrayList(); } deserializeReuse.ensureCapacity(thelist.size); - for(int i = 0; i < thelist.size; i++) { - if (i+1 > deserializeReuse.size()) { - deserializeReuse.add(this.getElementType().deserialize(null, iprot)); + for (int i = 0; i < thelist.size; i++) { + if (i + 1 > deserializeReuse.size()) { + deserializeReuse.add(getElementType().deserialize(null, iprot)); } else { - deserializeReuse.set(i, - this.getElementType().deserialize(deserializeReuse.get(i), iprot)); + deserializeReuse.set(i, getElementType().deserialize( + deserializeReuse.get(i), iprot)); } } - // in theory, the below call isn't needed in non thrift_mode, but let's not get too crazy + // in theory, the below call isn't needed in non thrift_mode, but let's not + // get too crazy iprot.readListEnd(); return deserializeReuse; } @Override - public void serialize(Object o, ObjectInspector oi, TProtocol oprot) throws TException, SerDeException, NoSuchFieldException,IllegalAccessException { - ListObjectInspector loi = (ListObjectInspector)oi; - ObjectInspector elementObjectInspector = loi.getListElementObjectInspector(); - DynamicSerDeTypeBase mt = this.getElementType(); + public void serialize(Object o, ObjectInspector oi, TProtocol oprot) + throws TException, SerDeException, NoSuchFieldException, + IllegalAccessException { + ListObjectInspector loi = (ListObjectInspector) oi; + ObjectInspector elementObjectInspector = loi + .getListElementObjectInspector(); + DynamicSerDeTypeBase mt = getElementType(); - org.apache.hadoop.hive.serde2.thrift.WriteNullsProtocol nullProtocol = - (oprot instanceof org.apache.hadoop.hive.serde2.thrift.WriteNullsProtocol) - ? (org.apache.hadoop.hive.serde2.thrift.WriteNullsProtocol)oprot - : null; - + org.apache.hadoop.hive.serde2.thrift.WriteNullsProtocol nullProtocol = (oprot instanceof org.apache.hadoop.hive.serde2.thrift.WriteNullsProtocol) ? (org.apache.hadoop.hive.serde2.thrift.WriteNullsProtocol) oprot + : null; + if (o instanceof List) { - List list = (List)o; + List list = (List) o; oprot.writeListBegin(new TList(mt.getType(), list.size())); - for (Object element: list) { + for (Object element : list) { if (element == null) { - assert(nullProtocol != null); + assert (nullProtocol != null); nullProtocol.writeNull(); } else { mt.serialize(element, elementObjectInspector, oprot); } } } else { - Object[] list = (Object[])o; - oprot.writeListBegin(new TList(mt.getType(),list.length)); - for (Object element: list) { + Object[] list = (Object[]) o; + oprot.writeListBegin(new TList(mt.getType(), list.length)); + for (Object element : list) { if (element == null && nullProtocol != null) { - assert(nullProtocol != null); + assert (nullProtocol != null); nullProtocol.writeNull(); } else { mt.serialize(element, elementObjectInspector, oprot); } } } - // in theory, the below call isn't needed in non thrift_mode, but let's not get too crazy + // in theory, the below call isn't needed in non thrift_mode, but let's not + // get too crazy oprot.writeListEnd(); } + @Override public byte getType() { return TType.LIST; } Index: serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/thrift_grammar.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/thrift_grammar.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/thrift_grammar.java (working copy) @@ -1,95 +1,103 @@ /* Generated By:JJTree&JavaCC: Do not edit this line. thrift_grammar.java */ package org.apache.hadoop.hive.serde2.dynamic_type; -import java.util.*; -import java.io.*; -import java.net.*; -import org.apache.thrift.protocol.*; -import org.apache.thrift.transport.*; -import org.apache.hadoop.hive.serde2.dynamic_type.*; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; -public class thrift_grammar/*@bgen(jjtree)*/implements thrift_grammarTreeConstants, thrift_grammarConstants {/*@bgen(jjtree)*/ +public class thrift_grammar/* @bgen(jjtree) */implements + thrift_grammarTreeConstants, thrift_grammarConstants {/* @bgen(jjtree) */ protected JJTthrift_grammarState jjtree = new JJTthrift_grammarState(); - private List include_path = null; + private List include_path = null; - // for computing the autogenerated field ids in thrift - private int field_val; + // for computing the autogenerated field ids in thrift + private int field_val; - // store types and tables - // separately because one cannot use a table (ie service.method) as a Struct like type. - protected Map types; - protected Map tables; + // store types and tables + // separately because one cannot use a table (ie service.method) as a Struct + // like type. + protected Map types; + protected Map tables; - // system include path - final private static String default_include_path[] = { "/usr/local/include","/usr/include","/usr/local/include/thrift/if","/usr/local/include/fb303/if" }; + // system include path + final private static String default_include_path[] = { "/usr/local/include", + "/usr/include", "/usr/local/include/thrift/if", + "/usr/local/include/fb303/if" }; - // need three params to differentiate between this and 2 param method auto generated since - // some calls in the autogenerated code use null param for 2nd param and thus ambiguous. - protected thrift_grammar(InputStream is, List include_path, boolean junk) { - this(is,null); - this.types = new HashMap () ; - this.tables = new HashMap () ; - this.include_path = include_path; - this.field_val = -1; - } + // need three params to differentiate between this and 2 param method auto + // generated since + // some calls in the autogenerated code use null param for 2nd param and thus + // ambiguous. + protected thrift_grammar(InputStream is, List include_path, + boolean junk) { + this(is, null); + types = new HashMap(); + tables = new HashMap(); + this.include_path = include_path; + field_val = -1; + } - // find the file on the include path - private static File findFile(String fname, List include_path) { - for(String path: include_path) { - final String full = path + "/" + fname; - File f = new File(full); - if(f.exists()) { - return f; - } - } - return null; + // find the file on the include path + private static File findFile(String fname, List include_path) { + for (String path : include_path) { + final String full = path + "/" + fname; + File f = new File(full); + if (f.exists()) { + return f; + } } + return null; + } - public static void main(String args[]) { - String filename = null; - List include_path = new ArrayList(); + public static void main(String args[]) { + String filename = null; + List include_path = new ArrayList(); - for(String path: default_include_path) { - include_path.add(path); - } - for(int i = 0; i < args.length; i++) { - String arg = args[i]; - if(arg.equals("--include") && i + 1 < args.length) { - include_path.add(args[++i]); - } - if(arg.equals("--file") && i + 1 < args.length) { - filename = args[++i]; - } - } + for (String path : default_include_path) { + include_path.add(path); + } + for (int i = 0; i < args.length; i++) { + String arg = args[i]; + if (arg.equals("--include") && i + 1 < args.length) { + include_path.add(args[++i]); + } + if (arg.equals("--file") && i + 1 < args.length) { + filename = args[++i]; + } + } - InputStream is = System.in; - if(filename != null) { - try { - is = new FileInputStream(findFile(filename, include_path)); - } catch(IOException e) { - } - } - thrift_grammar t = new thrift_grammar(is,include_path,false); + InputStream is = System.in; + if (filename != null) { + try { + is = new FileInputStream(findFile(filename, include_path)); + } catch (IOException e) { + } + } + thrift_grammar t = new thrift_grammar(is, include_path, false); - try { - t.Start(); - } catch (Exception e) { - System.out.println("Parse error."); - System.out.println(e.getMessage()); - e.printStackTrace(); - } + try { + t.Start(); + } catch (Exception e) { + System.out.println("Parse error."); + System.out.println(e.getMessage()); + e.printStackTrace(); } + } final public SimpleNode Start() throws ParseException { - /*@bgen(jjtree) Start */ - DynamicSerDeStart jjtn000 = new DynamicSerDeStart(JJTSTART); - boolean jjtc000 = true; - jjtree.openNodeScope(jjtn000); + /* @bgen(jjtree) Start */ + DynamicSerDeStart jjtn000 = new DynamicSerDeStart(JJTSTART); + boolean jjtc000 = true; + jjtree.openNodeScope(jjtn000); try { HeaderList(); - label_1: - while (true) { - switch ((jj_ntk==-1)?jj_ntk():jj_ntk) { + label_1: while (true) { + switch ((jj_ntk == -1) ? jj_ntk() : jj_ntk) { case 59: case 60: CommaOrSemicolon(); @@ -99,7 +107,7 @@ ; } Definition(); - switch ((jj_ntk==-1)?jj_ntk():jj_ntk) { + switch ((jj_ntk == -1) ? jj_ntk() : jj_ntk) { case tok_const: case tok_senum: case tok_typedef: @@ -116,40 +124,55 @@ break label_1; } } - jjtree.closeNodeScope(jjtn000, true); - jjtc000 = false; - {if (true) return jjtn000;} + jjtree.closeNodeScope(jjtn000, true); + jjtc000 = false; + { + if (true) { + return jjtn000; + } + } } catch (Throwable jjte000) { - if (jjtc000) { - jjtree.clearNodeScope(jjtn000); - jjtc000 = false; - } else { - jjtree.popNode(); - } - if (jjte000 instanceof RuntimeException) { - {if (true) throw (RuntimeException)jjte000;} - } - if (jjte000 instanceof ParseException) { - {if (true) throw (ParseException)jjte000;} - } - {if (true) throw (Error)jjte000;} + if (jjtc000) { + jjtree.clearNodeScope(jjtn000); + jjtc000 = false; + } else { + jjtree.popNode(); + } + if (jjte000 instanceof RuntimeException) { + { + if (true) { + throw (RuntimeException) jjte000; + } + } + } + if (jjte000 instanceof ParseException) { + { + if (true) { + throw (ParseException) jjte000; + } + } + } + { + if (true) { + throw (Error) jjte000; + } + } } finally { - if (jjtc000) { - jjtree.closeNodeScope(jjtn000, true); + if (jjtc000) { + jjtree.closeNodeScope(jjtn000, true); + } } - } throw new Error("Missing return statement in function"); } final public SimpleNode HeaderList() throws ParseException { - /*@bgen(jjtree) HeaderList */ - DynamicSerDeHeaderList jjtn000 = new DynamicSerDeHeaderList(JJTHEADERLIST); - boolean jjtc000 = true; - jjtree.openNodeScope(jjtn000); + /* @bgen(jjtree) HeaderList */ + DynamicSerDeHeaderList jjtn000 = new DynamicSerDeHeaderList(JJTHEADERLIST); + boolean jjtc000 = true; + jjtree.openNodeScope(jjtn000); try { - label_2: - while (true) { - switch ((jj_ntk==-1)?jj_ntk():jj_ntk) { + label_2: while (true) { + switch ((jj_ntk == -1) ? jj_ntk() : jj_ntk) { case tok_namespace: case tok_cpp_namespace: case tok_cpp_include: @@ -172,43 +195,63 @@ } Header(); } - jjtree.closeNodeScope(jjtn000, true); - jjtc000 = false; - {if (true) return jjtn000;} + jjtree.closeNodeScope(jjtn000, true); + jjtc000 = false; + { + if (true) { + return jjtn000; + } + } } catch (Throwable jjte000) { - if (jjtc000) { - jjtree.clearNodeScope(jjtn000); - jjtc000 = false; - } else { - jjtree.popNode(); - } - if (jjte000 instanceof RuntimeException) { - {if (true) throw (RuntimeException)jjte000;} - } - if (jjte000 instanceof ParseException) { - {if (true) throw (ParseException)jjte000;} - } - {if (true) throw (Error)jjte000;} + if (jjtc000) { + jjtree.clearNodeScope(jjtn000); + jjtc000 = false; + } else { + jjtree.popNode(); + } + if (jjte000 instanceof RuntimeException) { + { + if (true) { + throw (RuntimeException) jjte000; + } + } + } + if (jjte000 instanceof ParseException) { + { + if (true) { + throw (ParseException) jjte000; + } + } + } + { + if (true) { + throw (Error) jjte000; + } + } } finally { - if (jjtc000) { - jjtree.closeNodeScope(jjtn000, true); + if (jjtc000) { + jjtree.closeNodeScope(jjtn000, true); + } } - } throw new Error("Missing return statement in function"); } final public SimpleNode Header() throws ParseException { - /*@bgen(jjtree) Header */ - DynamicSerDeHeader jjtn000 = new DynamicSerDeHeader(JJTHEADER); - boolean jjtc000 = true; - jjtree.openNodeScope(jjtn000); + /* @bgen(jjtree) Header */ + DynamicSerDeHeader jjtn000 = new DynamicSerDeHeader(JJTHEADER); + boolean jjtc000 = true; + jjtree.openNodeScope(jjtn000); try { - switch ((jj_ntk==-1)?jj_ntk():jj_ntk) { + switch ((jj_ntk == -1) ? jj_ntk() : jj_ntk) { case tok_include: Include(); - jjtree.closeNodeScope(jjtn000, true); - jjtc000 = false; - {if (true) return jjtn000;} + jjtree.closeNodeScope(jjtn000, true); + jjtc000 = false; + { + if (true) { + return jjtn000; + } + } break; case tok_namespace: case tok_cpp_namespace: @@ -224,9 +267,13 @@ case tok_smalltalk_prefix: case tok_xsd_namespace: Namespace(); - jjtree.closeNodeScope(jjtn000, true); - jjtc000 = false; - {if (true) return jjtn000;} + jjtree.closeNodeScope(jjtn000, true); + jjtc000 = false; + { + if (true) { + return jjtn000; + } + } break; default: jj_la1[3] = jj_gen; @@ -234,125 +281,189 @@ throw new ParseException(); } } catch (Throwable jjte000) { - if (jjtc000) { - jjtree.clearNodeScope(jjtn000); - jjtc000 = false; - } else { - jjtree.popNode(); - } - if (jjte000 instanceof RuntimeException) { - {if (true) throw (RuntimeException)jjte000;} - } - if (jjte000 instanceof ParseException) { - {if (true) throw (ParseException)jjte000;} - } - {if (true) throw (Error)jjte000;} + if (jjtc000) { + jjtree.clearNodeScope(jjtn000); + jjtc000 = false; + } else { + jjtree.popNode(); + } + if (jjte000 instanceof RuntimeException) { + { + if (true) { + throw (RuntimeException) jjte000; + } + } + } + if (jjte000 instanceof ParseException) { + { + if (true) { + throw (ParseException) jjte000; + } + } + } + { + if (true) { + throw (Error) jjte000; + } + } } finally { - if (jjtc000) { - jjtree.closeNodeScope(jjtn000, true); + if (jjtc000) { + jjtree.closeNodeScope(jjtn000, true); + } } - } throw new Error("Missing return statement in function"); } final public SimpleNode Namespace() throws ParseException { - /*@bgen(jjtree) Namespace */ - DynamicSerDeNamespace jjtn000 = new DynamicSerDeNamespace(JJTNAMESPACE); - boolean jjtc000 = true; - jjtree.openNodeScope(jjtn000); + /* @bgen(jjtree) Namespace */ + DynamicSerDeNamespace jjtn000 = new DynamicSerDeNamespace(JJTNAMESPACE); + boolean jjtc000 = true; + jjtree.openNodeScope(jjtn000); try { - switch ((jj_ntk==-1)?jj_ntk():jj_ntk) { + switch ((jj_ntk == -1) ? jj_ntk() : jj_ntk) { case tok_namespace: jj_consume_token(tok_namespace); jj_consume_token(IDENTIFIER); jj_consume_token(IDENTIFIER); - jjtree.closeNodeScope(jjtn000, true); - jjtc000 = false; - {if (true) return jjtn000;} + jjtree.closeNodeScope(jjtn000, true); + jjtc000 = false; + { + if (true) { + return jjtn000; + } + } break; case tok_cpp_namespace: jj_consume_token(tok_cpp_namespace); jj_consume_token(IDENTIFIER); - jjtree.closeNodeScope(jjtn000, true); - jjtc000 = false; - {if (true) return jjtn000;} + jjtree.closeNodeScope(jjtn000, true); + jjtc000 = false; + { + if (true) { + return jjtn000; + } + } break; case tok_cpp_include: jj_consume_token(tok_cpp_include); jj_consume_token(tok_literal); - jjtree.closeNodeScope(jjtn000, true); - jjtc000 = false; - {if (true) return jjtn000;} + jjtree.closeNodeScope(jjtn000, true); + jjtc000 = false; + { + if (true) { + return jjtn000; + } + } break; case tok_php_namespace: jj_consume_token(tok_php_namespace); jj_consume_token(IDENTIFIER); - jjtree.closeNodeScope(jjtn000, true); - jjtc000 = false; - {if (true) return jjtn000;} + jjtree.closeNodeScope(jjtn000, true); + jjtc000 = false; + { + if (true) { + return jjtn000; + } + } break; case tok_py_module: jj_consume_token(tok_py_module); jj_consume_token(IDENTIFIER); - jjtree.closeNodeScope(jjtn000, true); - jjtc000 = false; - {if (true) return jjtn000;} + jjtree.closeNodeScope(jjtn000, true); + jjtc000 = false; + { + if (true) { + return jjtn000; + } + } break; case tok_perl_package: jj_consume_token(tok_perl_package); jj_consume_token(IDENTIFIER); - jjtree.closeNodeScope(jjtn000, true); - jjtc000 = false; - {if (true) return jjtn000;} + jjtree.closeNodeScope(jjtn000, true); + jjtc000 = false; + { + if (true) { + return jjtn000; + } + } break; case tok_ruby_namespace: jj_consume_token(tok_ruby_namespace); jj_consume_token(IDENTIFIER); - jjtree.closeNodeScope(jjtn000, true); - jjtc000 = false; - {if (true) return jjtn000;} + jjtree.closeNodeScope(jjtn000, true); + jjtc000 = false; + { + if (true) { + return jjtn000; + } + } break; case tok_smalltalk_category: jj_consume_token(tok_smalltalk_category); jj_consume_token(tok_st_identifier); - jjtree.closeNodeScope(jjtn000, true); - jjtc000 = false; - {if (true) return jjtn000;} + jjtree.closeNodeScope(jjtn000, true); + jjtc000 = false; + { + if (true) { + return jjtn000; + } + } break; case tok_smalltalk_prefix: jj_consume_token(tok_smalltalk_prefix); jj_consume_token(IDENTIFIER); - jjtree.closeNodeScope(jjtn000, true); - jjtc000 = false; - {if (true) return jjtn000;} + jjtree.closeNodeScope(jjtn000, true); + jjtc000 = false; + { + if (true) { + return jjtn000; + } + } break; case tok_java_package: jj_consume_token(tok_java_package); jj_consume_token(IDENTIFIER); - jjtree.closeNodeScope(jjtn000, true); - jjtc000 = false; - {if (true) return jjtn000;} + jjtree.closeNodeScope(jjtn000, true); + jjtc000 = false; + { + if (true) { + return jjtn000; + } + } break; case tok_cocoa_prefix: jj_consume_token(tok_cocoa_prefix); jj_consume_token(IDENTIFIER); - jjtree.closeNodeScope(jjtn000, true); - jjtc000 = false; - {if (true) return jjtn000;} + jjtree.closeNodeScope(jjtn000, true); + jjtc000 = false; + { + if (true) { + return jjtn000; + } + } break; case tok_xsd_namespace: jj_consume_token(tok_xsd_namespace); jj_consume_token(tok_literal); - jjtree.closeNodeScope(jjtn000, true); - jjtc000 = false; - {if (true) return jjtn000;} + jjtree.closeNodeScope(jjtn000, true); + jjtc000 = false; + { + if (true) { + return jjtn000; + } + } break; case tok_csharp_namespace: jj_consume_token(tok_csharp_namespace); jj_consume_token(IDENTIFIER); - jjtree.closeNodeScope(jjtn000, true); - jjtc000 = false; - {if (true) return jjtn000;} + jjtree.closeNodeScope(jjtn000, true); + jjtc000 = false; + { + if (true) { + return jjtn000; + } + } break; default: jj_la1[4] = jj_gen; @@ -360,76 +471,93 @@ throw new ParseException(); } } finally { - if (jjtc000) { - jjtree.closeNodeScope(jjtn000, true); + if (jjtc000) { + jjtree.closeNodeScope(jjtn000, true); + } } - } throw new Error("Missing return statement in function"); } final public SimpleNode Include() throws ParseException { - /*@bgen(jjtree) Include */ - DynamicSerDeInclude jjtn000 = new DynamicSerDeInclude(JJTINCLUDE); - boolean jjtc000 = true; - jjtree.openNodeScope(jjtn000);String fname; - boolean found = false; + /* @bgen(jjtree) Include */ + DynamicSerDeInclude jjtn000 = new DynamicSerDeInclude(JJTINCLUDE); + boolean jjtc000 = true; + jjtree.openNodeScope(jjtn000); + String fname; + boolean found = false; try { jj_consume_token(tok_include); fname = jj_consume_token(tok_literal).image; - jjtree.closeNodeScope(jjtn000, true); - jjtc000 = false; - // bugbug somewhat fragile below substring expression - fname = fname.substring(1,fname.length() - 1); + jjtree.closeNodeScope(jjtn000, true); + jjtc000 = false; + // bugbug somewhat fragile below substring expression + fname = fname.substring(1, fname.length() - 1); - // try to find the file on the include path - File f = thrift_grammar.findFile(fname, this.include_path); - if(f != null) { + // try to find the file on the include path + File f = thrift_grammar.findFile(fname, include_path); + if (f != null) { found = true; try { - FileInputStream fis = new FileInputStream(f); - thrift_grammar t = new thrift_grammar(fis,this.include_path, false); - t.Start(); - fis.close(); - found = true; - // add in what we found to our type and table tables. - this.tables.putAll(t.tables); - this.types.putAll(t.types); + FileInputStream fis = new FileInputStream(f); + thrift_grammar t = new thrift_grammar(fis, include_path, false); + t.Start(); + fis.close(); + found = true; + // add in what we found to our type and table tables. + tables.putAll(t.tables); + types.putAll(t.types); } catch (Exception e) { - System.out.println("File: " + fname + " - Oops."); - System.out.println(e.getMessage()); - e.printStackTrace(); + System.out.println("File: " + fname + " - Oops."); + System.out.println(e.getMessage()); + e.printStackTrace(); } - } - if(!found) { - {if (true) throw new RuntimeException("include file not found: " + fname);} - } - {if (true) return jjtn000;} + } + if (!found) { + { + if (true) { + throw new RuntimeException("include file not found: " + fname); + } + } + } + { + if (true) { + return jjtn000; + } + } } finally { - if (jjtc000) { - jjtree.closeNodeScope(jjtn000, true); - } + if (jjtc000) { + jjtree.closeNodeScope(jjtn000, true); + } } throw new Error("Missing return statement in function"); } final public SimpleNode Definition() throws ParseException { - /*@bgen(jjtree) Definition */ - DynamicSerDeDefinition jjtn000 = new DynamicSerDeDefinition(JJTDEFINITION); - boolean jjtc000 = true; - jjtree.openNodeScope(jjtn000); + /* @bgen(jjtree) Definition */ + DynamicSerDeDefinition jjtn000 = new DynamicSerDeDefinition(JJTDEFINITION); + boolean jjtc000 = true; + jjtree.openNodeScope(jjtn000); try { - switch ((jj_ntk==-1)?jj_ntk():jj_ntk) { + switch ((jj_ntk == -1) ? jj_ntk() : jj_ntk) { case tok_const: Const(); - jjtree.closeNodeScope(jjtn000, true); - jjtc000 = false; - {if (true) return jjtn000;} + jjtree.closeNodeScope(jjtn000, true); + jjtc000 = false; + { + if (true) { + return jjtn000; + } + } break; case tok_service: Service(); - jjtree.closeNodeScope(jjtn000, true); - jjtc000 = false; - {if (true) return jjtn000;} + jjtree.closeNodeScope(jjtn000, true); + jjtc000 = false; + { + if (true) { + return jjtn000; + } + } break; case tok_senum: case tok_typedef: @@ -437,9 +565,13 @@ case tok_exception: case tok_enum: TypeDefinition(); - jjtree.closeNodeScope(jjtn000, true); - jjtc000 = false; - {if (true) return jjtn000;} + jjtree.closeNodeScope(jjtn000, true); + jjtc000 = false; + { + if (true) { + return jjtn000; + } + } break; default: jj_la1[5] = jj_gen; @@ -447,63 +579,96 @@ throw new ParseException(); } } catch (Throwable jjte000) { - if (jjtc000) { - jjtree.clearNodeScope(jjtn000); - jjtc000 = false; - } else { - jjtree.popNode(); - } - if (jjte000 instanceof RuntimeException) { - {if (true) throw (RuntimeException)jjte000;} - } - if (jjte000 instanceof ParseException) { - {if (true) throw (ParseException)jjte000;} - } - {if (true) throw (Error)jjte000;} + if (jjtc000) { + jjtree.clearNodeScope(jjtn000); + jjtc000 = false; + } else { + jjtree.popNode(); + } + if (jjte000 instanceof RuntimeException) { + { + if (true) { + throw (RuntimeException) jjte000; + } + } + } + if (jjte000 instanceof ParseException) { + { + if (true) { + throw (ParseException) jjte000; + } + } + } + { + if (true) { + throw (Error) jjte000; + } + } } finally { - if (jjtc000) { - jjtree.closeNodeScope(jjtn000, true); + if (jjtc000) { + jjtree.closeNodeScope(jjtn000, true); + } } - } throw new Error("Missing return statement in function"); } final public SimpleNode TypeDefinition() throws ParseException { - /*@bgen(jjtree) TypeDefinition */ - DynamicSerDeTypeDefinition jjtn000 = new DynamicSerDeTypeDefinition(JJTTYPEDEFINITION); - boolean jjtc000 = true; - jjtree.openNodeScope(jjtn000); + /* @bgen(jjtree) TypeDefinition */ + DynamicSerDeTypeDefinition jjtn000 = new DynamicSerDeTypeDefinition( + JJTTYPEDEFINITION); + boolean jjtc000 = true; + jjtree.openNodeScope(jjtn000); try { - switch ((jj_ntk==-1)?jj_ntk():jj_ntk) { + switch ((jj_ntk == -1) ? jj_ntk() : jj_ntk) { case tok_typedef: Typedef(); - jjtree.closeNodeScope(jjtn000, true); - jjtc000 = false; - {if (true) return jjtn000;} + jjtree.closeNodeScope(jjtn000, true); + jjtc000 = false; + { + if (true) { + return jjtn000; + } + } break; case tok_enum: Enum(); - jjtree.closeNodeScope(jjtn000, true); - jjtc000 = false; - {if (true) return jjtn000;} + jjtree.closeNodeScope(jjtn000, true); + jjtc000 = false; + { + if (true) { + return jjtn000; + } + } break; case tok_senum: Senum(); - jjtree.closeNodeScope(jjtn000, true); - jjtc000 = false; - {if (true) return jjtn000;} + jjtree.closeNodeScope(jjtn000, true); + jjtc000 = false; + { + if (true) { + return jjtn000; + } + } break; case tok_struct: Struct(); - jjtree.closeNodeScope(jjtn000, true); - jjtc000 = false; - {if (true) return jjtn000;} + jjtree.closeNodeScope(jjtn000, true); + jjtc000 = false; + { + if (true) { + return jjtn000; + } + } break; case tok_exception: Xception(); - jjtree.closeNodeScope(jjtn000, true); - jjtc000 = false; - {if (true) return jjtn000;} + jjtree.closeNodeScope(jjtn000, true); + jjtc000 = false; + { + if (true) { + return jjtn000; + } + } break; default: jj_la1[6] = jj_gen; @@ -511,78 +676,107 @@ throw new ParseException(); } } catch (Throwable jjte000) { - if (jjtc000) { - jjtree.clearNodeScope(jjtn000); - jjtc000 = false; - } else { - jjtree.popNode(); - } - if (jjte000 instanceof RuntimeException) { - {if (true) throw (RuntimeException)jjte000;} - } - if (jjte000 instanceof ParseException) { - {if (true) throw (ParseException)jjte000;} - } - {if (true) throw (Error)jjte000;} + if (jjtc000) { + jjtree.clearNodeScope(jjtn000); + jjtc000 = false; + } else { + jjtree.popNode(); + } + if (jjte000 instanceof RuntimeException) { + { + if (true) { + throw (RuntimeException) jjte000; + } + } + } + if (jjte000 instanceof ParseException) { + { + if (true) { + throw (ParseException) jjte000; + } + } + } + { + if (true) { + throw (Error) jjte000; + } + } } finally { - if (jjtc000) { - jjtree.closeNodeScope(jjtn000, true); + if (jjtc000) { + jjtree.closeNodeScope(jjtn000, true); + } } - } throw new Error("Missing return statement in function"); } final public DynamicSerDeTypedef Typedef() throws ParseException { - /*@bgen(jjtree) Typedef */ - DynamicSerDeTypedef jjtn000 = new DynamicSerDeTypedef(JJTTYPEDEF); - boolean jjtc000 = true; - jjtree.openNodeScope(jjtn000); + /* @bgen(jjtree) Typedef */ + DynamicSerDeTypedef jjtn000 = new DynamicSerDeTypedef(JJTTYPEDEF); + boolean jjtc000 = true; + jjtree.openNodeScope(jjtn000); try { jj_consume_token(tok_typedef); DefinitionType(); jjtn000.name = jj_consume_token(IDENTIFIER).image; jjtree.closeNodeScope(jjtn000, true); jjtc000 = false; - // store the type for later retrieval - this.types.put(jjtn000.name, jjtn000); - {if (true) return jjtn000;} + // store the type for later retrieval + types.put(jjtn000.name, jjtn000); + { + if (true) { + return jjtn000; + } + } } catch (Throwable jjte000) { - if (jjtc000) { - jjtree.clearNodeScope(jjtn000); - jjtc000 = false; - } else { - jjtree.popNode(); - } - if (jjte000 instanceof RuntimeException) { - {if (true) throw (RuntimeException)jjte000;} - } - if (jjte000 instanceof ParseException) { - {if (true) throw (ParseException)jjte000;} - } - {if (true) throw (Error)jjte000;} + if (jjtc000) { + jjtree.clearNodeScope(jjtn000); + jjtc000 = false; + } else { + jjtree.popNode(); + } + if (jjte000 instanceof RuntimeException) { + { + if (true) { + throw (RuntimeException) jjte000; + } + } + } + if (jjte000 instanceof ParseException) { + { + if (true) { + throw (ParseException) jjte000; + } + } + } + { + if (true) { + throw (Error) jjte000; + } + } } finally { - if (jjtc000) { - jjtree.closeNodeScope(jjtn000, true); + if (jjtc000) { + jjtree.closeNodeScope(jjtn000, true); + } } - } throw new Error("Missing return statement in function"); } -// returning void because we ignore this production. + // returning void because we ignore this production. final public void CommaOrSemicolon() throws ParseException { - /*@bgen(jjtree) CommaOrSemicolon */ - DynamicSerDeCommaOrSemicolon jjtn000 = new DynamicSerDeCommaOrSemicolon(JJTCOMMAORSEMICOLON); - boolean jjtc000 = true; - jjtree.openNodeScope(jjtn000); + /* @bgen(jjtree) CommaOrSemicolon */ + DynamicSerDeCommaOrSemicolon jjtn000 = new DynamicSerDeCommaOrSemicolon( + JJTCOMMAORSEMICOLON); + boolean jjtc000 = true; + jjtree.openNodeScope(jjtn000); try { - switch ((jj_ntk==-1)?jj_ntk():jj_ntk) { + switch ((jj_ntk == -1) ? jj_ntk() : jj_ntk) { case 59: jj_consume_token(59); break; case 60: jj_consume_token(60); - jjtree.closeNodeScope(jjtn000, true); - jjtc000 = false; + jjtree.closeNodeScope(jjtn000, true); + jjtc000 = false; break; default: @@ -591,17 +785,17 @@ throw new ParseException(); } } finally { - if (jjtc000) { - jjtree.closeNodeScope(jjtn000, true); + if (jjtc000) { + jjtree.closeNodeScope(jjtn000, true); + } } - } } final public SimpleNode Enum() throws ParseException { - /*@bgen(jjtree) Enum */ - DynamicSerDeEnum jjtn000 = new DynamicSerDeEnum(JJTENUM); - boolean jjtc000 = true; - jjtree.openNodeScope(jjtn000); + /* @bgen(jjtree) Enum */ + DynamicSerDeEnum jjtn000 = new DynamicSerDeEnum(JJTENUM); + boolean jjtc000 = true; + jjtree.openNodeScope(jjtn000); try { jj_consume_token(tok_enum); jj_consume_token(IDENTIFIER); @@ -610,39 +804,55 @@ jj_consume_token(62); jjtree.closeNodeScope(jjtn000, true); jjtc000 = false; - {if (true) return jjtn000;} + { + if (true) { + return jjtn000; + } + } } catch (Throwable jjte000) { - if (jjtc000) { - jjtree.clearNodeScope(jjtn000); - jjtc000 = false; - } else { - jjtree.popNode(); - } - if (jjte000 instanceof RuntimeException) { - {if (true) throw (RuntimeException)jjte000;} - } - if (jjte000 instanceof ParseException) { - {if (true) throw (ParseException)jjte000;} - } - {if (true) throw (Error)jjte000;} + if (jjtc000) { + jjtree.clearNodeScope(jjtn000); + jjtc000 = false; + } else { + jjtree.popNode(); + } + if (jjte000 instanceof RuntimeException) { + { + if (true) { + throw (RuntimeException) jjte000; + } + } + } + if (jjte000 instanceof ParseException) { + { + if (true) { + throw (ParseException) jjte000; + } + } + } + { + if (true) { + throw (Error) jjte000; + } + } } finally { - if (jjtc000) { - jjtree.closeNodeScope(jjtn000, true); + if (jjtc000) { + jjtree.closeNodeScope(jjtn000, true); + } } - } throw new Error("Missing return statement in function"); } final public SimpleNode EnumDefList() throws ParseException { - /*@bgen(jjtree) EnumDefList */ - DynamicSerDeEnumDefList jjtn000 = new DynamicSerDeEnumDefList(JJTENUMDEFLIST); - boolean jjtc000 = true; - jjtree.openNodeScope(jjtn000); + /* @bgen(jjtree) EnumDefList */ + DynamicSerDeEnumDefList jjtn000 = new DynamicSerDeEnumDefList( + JJTENUMDEFLIST); + boolean jjtc000 = true; + jjtree.openNodeScope(jjtn000); try { - label_3: - while (true) { + label_3: while (true) { EnumDef(); - switch ((jj_ntk==-1)?jj_ntk():jj_ntk) { + switch ((jj_ntk == -1) ? jj_ntk() : jj_ntk) { case IDENTIFIER: ; break; @@ -653,37 +863,53 @@ } jjtree.closeNodeScope(jjtn000, true); jjtc000 = false; - {if (true) return jjtn000;} + { + if (true) { + return jjtn000; + } + } } catch (Throwable jjte000) { - if (jjtc000) { - jjtree.clearNodeScope(jjtn000); - jjtc000 = false; - } else { - jjtree.popNode(); - } - if (jjte000 instanceof RuntimeException) { - {if (true) throw (RuntimeException)jjte000;} - } - if (jjte000 instanceof ParseException) { - {if (true) throw (ParseException)jjte000;} - } - {if (true) throw (Error)jjte000;} + if (jjtc000) { + jjtree.clearNodeScope(jjtn000); + jjtc000 = false; + } else { + jjtree.popNode(); + } + if (jjte000 instanceof RuntimeException) { + { + if (true) { + throw (RuntimeException) jjte000; + } + } + } + if (jjte000 instanceof ParseException) { + { + if (true) { + throw (ParseException) jjte000; + } + } + } + { + if (true) { + throw (Error) jjte000; + } + } } finally { - if (jjtc000) { - jjtree.closeNodeScope(jjtn000, true); + if (jjtc000) { + jjtree.closeNodeScope(jjtn000, true); + } } - } throw new Error("Missing return statement in function"); } final public SimpleNode EnumDef() throws ParseException { - /*@bgen(jjtree) EnumDef */ - DynamicSerDeEnumDef jjtn000 = new DynamicSerDeEnumDef(JJTENUMDEF); - boolean jjtc000 = true; - jjtree.openNodeScope(jjtn000); + /* @bgen(jjtree) EnumDef */ + DynamicSerDeEnumDef jjtn000 = new DynamicSerDeEnumDef(JJTENUMDEF); + boolean jjtc000 = true; + jjtree.openNodeScope(jjtn000); try { jj_consume_token(IDENTIFIER); - switch ((jj_ntk==-1)?jj_ntk():jj_ntk) { + switch ((jj_ntk == -1) ? jj_ntk() : jj_ntk) { case 63: jj_consume_token(63); jj_consume_token(tok_int_constant); @@ -692,7 +918,7 @@ jj_la1[9] = jj_gen; ; } - switch ((jj_ntk==-1)?jj_ntk():jj_ntk) { + switch ((jj_ntk == -1) ? jj_ntk() : jj_ntk) { case 59: case 60: CommaOrSemicolon(); @@ -703,34 +929,50 @@ } jjtree.closeNodeScope(jjtn000, true); jjtc000 = false; - {if (true) return jjtn000;} + { + if (true) { + return jjtn000; + } + } } catch (Throwable jjte000) { - if (jjtc000) { - jjtree.clearNodeScope(jjtn000); - jjtc000 = false; - } else { - jjtree.popNode(); - } - if (jjte000 instanceof RuntimeException) { - {if (true) throw (RuntimeException)jjte000;} - } - if (jjte000 instanceof ParseException) { - {if (true) throw (ParseException)jjte000;} - } - {if (true) throw (Error)jjte000;} + if (jjtc000) { + jjtree.clearNodeScope(jjtn000); + jjtc000 = false; + } else { + jjtree.popNode(); + } + if (jjte000 instanceof RuntimeException) { + { + if (true) { + throw (RuntimeException) jjte000; + } + } + } + if (jjte000 instanceof ParseException) { + { + if (true) { + throw (ParseException) jjte000; + } + } + } + { + if (true) { + throw (Error) jjte000; + } + } } finally { - if (jjtc000) { - jjtree.closeNodeScope(jjtn000, true); + if (jjtc000) { + jjtree.closeNodeScope(jjtn000, true); + } } - } throw new Error("Missing return statement in function"); } final public SimpleNode Senum() throws ParseException { - /*@bgen(jjtree) Senum */ - DynamicSerDeSenum jjtn000 = new DynamicSerDeSenum(JJTSENUM); - boolean jjtc000 = true; - jjtree.openNodeScope(jjtn000); + /* @bgen(jjtree) Senum */ + DynamicSerDeSenum jjtn000 = new DynamicSerDeSenum(JJTSENUM); + boolean jjtc000 = true; + jjtree.openNodeScope(jjtn000); try { jj_consume_token(tok_senum); jj_consume_token(IDENTIFIER); @@ -739,39 +981,55 @@ jj_consume_token(62); jjtree.closeNodeScope(jjtn000, true); jjtc000 = false; - {if (true) return jjtn000;} + { + if (true) { + return jjtn000; + } + } } catch (Throwable jjte000) { - if (jjtc000) { - jjtree.clearNodeScope(jjtn000); - jjtc000 = false; - } else { - jjtree.popNode(); - } - if (jjte000 instanceof RuntimeException) { - {if (true) throw (RuntimeException)jjte000;} - } - if (jjte000 instanceof ParseException) { - {if (true) throw (ParseException)jjte000;} - } - {if (true) throw (Error)jjte000;} + if (jjtc000) { + jjtree.clearNodeScope(jjtn000); + jjtc000 = false; + } else { + jjtree.popNode(); + } + if (jjte000 instanceof RuntimeException) { + { + if (true) { + throw (RuntimeException) jjte000; + } + } + } + if (jjte000 instanceof ParseException) { + { + if (true) { + throw (ParseException) jjte000; + } + } + } + { + if (true) { + throw (Error) jjte000; + } + } } finally { - if (jjtc000) { - jjtree.closeNodeScope(jjtn000, true); + if (jjtc000) { + jjtree.closeNodeScope(jjtn000, true); + } } - } throw new Error("Missing return statement in function"); } final public SimpleNode SenumDefList() throws ParseException { - /*@bgen(jjtree) SenumDefList */ - DynamicSerDeSenumDefList jjtn000 = new DynamicSerDeSenumDefList(JJTSENUMDEFLIST); - boolean jjtc000 = true; - jjtree.openNodeScope(jjtn000); + /* @bgen(jjtree) SenumDefList */ + DynamicSerDeSenumDefList jjtn000 = new DynamicSerDeSenumDefList( + JJTSENUMDEFLIST); + boolean jjtc000 = true; + jjtree.openNodeScope(jjtn000); try { - label_4: - while (true) { + label_4: while (true) { SenumDef(); - switch ((jj_ntk==-1)?jj_ntk():jj_ntk) { + switch ((jj_ntk == -1) ? jj_ntk() : jj_ntk) { case tok_literal: ; break; @@ -782,37 +1040,53 @@ } jjtree.closeNodeScope(jjtn000, true); jjtc000 = false; - {if (true) return jjtn000;} + { + if (true) { + return jjtn000; + } + } } catch (Throwable jjte000) { - if (jjtc000) { - jjtree.clearNodeScope(jjtn000); - jjtc000 = false; - } else { - jjtree.popNode(); - } - if (jjte000 instanceof RuntimeException) { - {if (true) throw (RuntimeException)jjte000;} - } - if (jjte000 instanceof ParseException) { - {if (true) throw (ParseException)jjte000;} - } - {if (true) throw (Error)jjte000;} + if (jjtc000) { + jjtree.clearNodeScope(jjtn000); + jjtc000 = false; + } else { + jjtree.popNode(); + } + if (jjte000 instanceof RuntimeException) { + { + if (true) { + throw (RuntimeException) jjte000; + } + } + } + if (jjte000 instanceof ParseException) { + { + if (true) { + throw (ParseException) jjte000; + } + } + } + { + if (true) { + throw (Error) jjte000; + } + } } finally { - if (jjtc000) { - jjtree.closeNodeScope(jjtn000, true); + if (jjtc000) { + jjtree.closeNodeScope(jjtn000, true); + } } - } throw new Error("Missing return statement in function"); } final public SimpleNode SenumDef() throws ParseException { - /*@bgen(jjtree) SenumDef */ - DynamicSerDeSenumDef jjtn000 = new DynamicSerDeSenumDef(JJTSENUMDEF); - boolean jjtc000 = true; - jjtree.openNodeScope(jjtn000); + /* @bgen(jjtree) SenumDef */ + DynamicSerDeSenumDef jjtn000 = new DynamicSerDeSenumDef(JJTSENUMDEF); + boolean jjtc000 = true; + jjtree.openNodeScope(jjtn000); try { jj_consume_token(tok_literal); - switch ((jj_ntk==-1)?jj_ntk():jj_ntk) { + switch ((jj_ntk == -1) ? jj_ntk() : jj_ntk) { case 59: case 60: CommaOrSemicolon(); @@ -823,41 +1097,57 @@ } jjtree.closeNodeScope(jjtn000, true); jjtc000 = false; - {if (true) return jjtn000;} + { + if (true) { + return jjtn000; + } + } } catch (Throwable jjte000) { - if (jjtc000) { - jjtree.clearNodeScope(jjtn000); - jjtc000 = false; - } else { - jjtree.popNode(); - } - if (jjte000 instanceof RuntimeException) { - {if (true) throw (RuntimeException)jjte000;} - } - if (jjte000 instanceof ParseException) { - {if (true) throw (ParseException)jjte000;} - } - {if (true) throw (Error)jjte000;} + if (jjtc000) { + jjtree.clearNodeScope(jjtn000); + jjtc000 = false; + } else { + jjtree.popNode(); + } + if (jjte000 instanceof RuntimeException) { + { + if (true) { + throw (RuntimeException) jjte000; + } + } + } + if (jjte000 instanceof ParseException) { + { + if (true) { + throw (ParseException) jjte000; + } + } + } + { + if (true) { + throw (Error) jjte000; + } + } } finally { - if (jjtc000) { - jjtree.closeNodeScope(jjtn000, true); + if (jjtc000) { + jjtree.closeNodeScope(jjtn000, true); + } } - } throw new Error("Missing return statement in function"); } final public SimpleNode Const() throws ParseException { - /*@bgen(jjtree) Const */ - DynamicSerDeConst jjtn000 = new DynamicSerDeConst(JJTCONST); - boolean jjtc000 = true; - jjtree.openNodeScope(jjtn000); + /* @bgen(jjtree) Const */ + DynamicSerDeConst jjtn000 = new DynamicSerDeConst(JJTCONST); + boolean jjtc000 = true; + jjtree.openNodeScope(jjtn000); try { jj_consume_token(tok_const); FieldType(); jj_consume_token(IDENTIFIER); jj_consume_token(63); ConstValue(); - switch ((jj_ntk==-1)?jj_ntk():jj_ntk) { + switch ((jj_ntk == -1) ? jj_ntk() : jj_ntk) { case 59: case 60: CommaOrSemicolon(); @@ -868,71 +1158,91 @@ } jjtree.closeNodeScope(jjtn000, true); jjtc000 = false; - {if (true) return jjtn000;} + { + if (true) { + return jjtn000; + } + } } catch (Throwable jjte000) { - if (jjtc000) { - jjtree.clearNodeScope(jjtn000); - jjtc000 = false; - } else { - jjtree.popNode(); - } - if (jjte000 instanceof RuntimeException) { - {if (true) throw (RuntimeException)jjte000;} - } - if (jjte000 instanceof ParseException) { - {if (true) throw (ParseException)jjte000;} - } - {if (true) throw (Error)jjte000;} + if (jjtc000) { + jjtree.clearNodeScope(jjtn000); + jjtc000 = false; + } else { + jjtree.popNode(); + } + if (jjte000 instanceof RuntimeException) { + { + if (true) { + throw (RuntimeException) jjte000; + } + } + } + if (jjte000 instanceof ParseException) { + { + if (true) { + throw (ParseException) jjte000; + } + } + } + { + if (true) { + throw (Error) jjte000; + } + } } finally { - if (jjtc000) { - jjtree.closeNodeScope(jjtn000, true); + if (jjtc000) { + jjtree.closeNodeScope(jjtn000, true); + } } - } throw new Error("Missing return statement in function"); } final public SimpleNode ConstValue() throws ParseException { - /*@bgen(jjtree) ConstValue */ - DynamicSerDeConstValue jjtn000 = new DynamicSerDeConstValue(JJTCONSTVALUE); - boolean jjtc000 = true; - jjtree.openNodeScope(jjtn000); + /* @bgen(jjtree) ConstValue */ + DynamicSerDeConstValue jjtn000 = new DynamicSerDeConstValue(JJTCONSTVALUE); + boolean jjtc000 = true; + jjtree.openNodeScope(jjtn000); try { - switch ((jj_ntk==-1)?jj_ntk():jj_ntk) { + switch ((jj_ntk == -1) ? jj_ntk() : jj_ntk) { case tok_int_constant: jj_consume_token(tok_int_constant); - jjtree.closeNodeScope(jjtn000, true); - jjtc000 = false; + jjtree.closeNodeScope(jjtn000, true); + jjtc000 = false; break; case tok_double_constant: jj_consume_token(tok_double_constant); - jjtree.closeNodeScope(jjtn000, true); - jjtc000 = false; + jjtree.closeNodeScope(jjtn000, true); + jjtc000 = false; break; case tok_literal: jj_consume_token(tok_literal); - jjtree.closeNodeScope(jjtn000, true); - jjtc000 = false; + jjtree.closeNodeScope(jjtn000, true); + jjtc000 = false; break; case IDENTIFIER: jj_consume_token(IDENTIFIER); - jjtree.closeNodeScope(jjtn000, true); - jjtc000 = false; + jjtree.closeNodeScope(jjtn000, true); + jjtc000 = false; break; case 64: ConstList(); - jjtree.closeNodeScope(jjtn000, true); - jjtc000 = false; + jjtree.closeNodeScope(jjtn000, true); + jjtc000 = false; break; case 61: ConstMap(); - jjtree.closeNodeScope(jjtn000, true); - jjtc000 = false; - {if (true) return jjtn000;} + jjtree.closeNodeScope(jjtn000, true); + jjtc000 = false; + { + if (true) { + return jjtn000; + } + } break; default: jj_la1[14] = jj_gen; @@ -940,71 +1250,99 @@ throw new ParseException(); } } catch (Throwable jjte000) { - if (jjtc000) { - jjtree.clearNodeScope(jjtn000); - jjtc000 = false; - } else { - jjtree.popNode(); - } - if (jjte000 instanceof RuntimeException) { - {if (true) throw (RuntimeException)jjte000;} - } - if (jjte000 instanceof ParseException) { - {if (true) throw (ParseException)jjte000;} - } - {if (true) throw (Error)jjte000;} + if (jjtc000) { + jjtree.clearNodeScope(jjtn000); + jjtc000 = false; + } else { + jjtree.popNode(); + } + if (jjte000 instanceof RuntimeException) { + { + if (true) { + throw (RuntimeException) jjte000; + } + } + } + if (jjte000 instanceof ParseException) { + { + if (true) { + throw (ParseException) jjte000; + } + } + } + { + if (true) { + throw (Error) jjte000; + } + } } finally { - if (jjtc000) { - jjtree.closeNodeScope(jjtn000, true); + if (jjtc000) { + jjtree.closeNodeScope(jjtn000, true); + } } - } throw new Error("Missing return statement in function"); } final public SimpleNode ConstList() throws ParseException { - /*@bgen(jjtree) ConstList */ - DynamicSerDeConstList jjtn000 = new DynamicSerDeConstList(JJTCONSTLIST); - boolean jjtc000 = true; - jjtree.openNodeScope(jjtn000); + /* @bgen(jjtree) ConstList */ + DynamicSerDeConstList jjtn000 = new DynamicSerDeConstList(JJTCONSTLIST); + boolean jjtc000 = true; + jjtree.openNodeScope(jjtn000); try { jj_consume_token(64); ConstListContents(); jj_consume_token(65); jjtree.closeNodeScope(jjtn000, true); jjtc000 = false; - {if (true) return jjtn000;} + { + if (true) { + return jjtn000; + } + } } catch (Throwable jjte000) { - if (jjtc000) { - jjtree.clearNodeScope(jjtn000); - jjtc000 = false; - } else { - jjtree.popNode(); - } - if (jjte000 instanceof RuntimeException) { - {if (true) throw (RuntimeException)jjte000;} - } - if (jjte000 instanceof ParseException) { - {if (true) throw (ParseException)jjte000;} - } - {if (true) throw (Error)jjte000;} + if (jjtc000) { + jjtree.clearNodeScope(jjtn000); + jjtc000 = false; + } else { + jjtree.popNode(); + } + if (jjte000 instanceof RuntimeException) { + { + if (true) { + throw (RuntimeException) jjte000; + } + } + } + if (jjte000 instanceof ParseException) { + { + if (true) { + throw (ParseException) jjte000; + } + } + } + { + if (true) { + throw (Error) jjte000; + } + } } finally { - if (jjtc000) { - jjtree.closeNodeScope(jjtn000, true); + if (jjtc000) { + jjtree.closeNodeScope(jjtn000, true); + } } - } throw new Error("Missing return statement in function"); } final public SimpleNode ConstListContents() throws ParseException { - /*@bgen(jjtree) ConstListContents */ - DynamicSerDeConstListContents jjtn000 = new DynamicSerDeConstListContents(JJTCONSTLISTCONTENTS); - boolean jjtc000 = true; - jjtree.openNodeScope(jjtn000); + /* @bgen(jjtree) ConstListContents */ + DynamicSerDeConstListContents jjtn000 = new DynamicSerDeConstListContents( + JJTCONSTLISTCONTENTS); + boolean jjtc000 = true; + jjtree.openNodeScope(jjtn000); try { - label_5: - while (true) { + label_5: while (true) { ConstValue(); - switch ((jj_ntk==-1)?jj_ntk():jj_ntk) { + switch ((jj_ntk == -1) ? jj_ntk() : jj_ntk) { case 59: case 60: CommaOrSemicolon(); @@ -1013,7 +1351,7 @@ jj_la1[15] = jj_gen; ; } - switch ((jj_ntk==-1)?jj_ntk():jj_ntk) { + switch ((jj_ntk == -1) ? jj_ntk() : jj_ntk) { case tok_int_constant: case tok_double_constant: case IDENTIFIER: @@ -1029,82 +1367,114 @@ } jjtree.closeNodeScope(jjtn000, true); jjtc000 = false; - {if (true) return jjtn000;} + { + if (true) { + return jjtn000; + } + } } catch (Throwable jjte000) { - if (jjtc000) { - jjtree.clearNodeScope(jjtn000); - jjtc000 = false; - } else { - jjtree.popNode(); - } - if (jjte000 instanceof RuntimeException) { - {if (true) throw (RuntimeException)jjte000;} - } - if (jjte000 instanceof ParseException) { - {if (true) throw (ParseException)jjte000;} - } - {if (true) throw (Error)jjte000;} + if (jjtc000) { + jjtree.clearNodeScope(jjtn000); + jjtc000 = false; + } else { + jjtree.popNode(); + } + if (jjte000 instanceof RuntimeException) { + { + if (true) { + throw (RuntimeException) jjte000; + } + } + } + if (jjte000 instanceof ParseException) { + { + if (true) { + throw (ParseException) jjte000; + } + } + } + { + if (true) { + throw (Error) jjte000; + } + } } finally { - if (jjtc000) { - jjtree.closeNodeScope(jjtn000, true); + if (jjtc000) { + jjtree.closeNodeScope(jjtn000, true); + } } - } throw new Error("Missing return statement in function"); } final public SimpleNode ConstMap() throws ParseException { - /*@bgen(jjtree) ConstMap */ - DynamicSerDeConstMap jjtn000 = new DynamicSerDeConstMap(JJTCONSTMAP); - boolean jjtc000 = true; - jjtree.openNodeScope(jjtn000); + /* @bgen(jjtree) ConstMap */ + DynamicSerDeConstMap jjtn000 = new DynamicSerDeConstMap(JJTCONSTMAP); + boolean jjtc000 = true; + jjtree.openNodeScope(jjtn000); try { jj_consume_token(61); ConstMapContents(); jj_consume_token(62); jjtree.closeNodeScope(jjtn000, true); jjtc000 = false; - {if (true) return jjtn000;} + { + if (true) { + return jjtn000; + } + } } catch (Throwable jjte000) { - if (jjtc000) { - jjtree.clearNodeScope(jjtn000); - jjtc000 = false; - } else { - jjtree.popNode(); - } - if (jjte000 instanceof RuntimeException) { - {if (true) throw (RuntimeException)jjte000;} - } - if (jjte000 instanceof ParseException) { - {if (true) throw (ParseException)jjte000;} - } - {if (true) throw (Error)jjte000;} + if (jjtc000) { + jjtree.clearNodeScope(jjtn000); + jjtc000 = false; + } else { + jjtree.popNode(); + } + if (jjte000 instanceof RuntimeException) { + { + if (true) { + throw (RuntimeException) jjte000; + } + } + } + if (jjte000 instanceof ParseException) { + { + if (true) { + throw (ParseException) jjte000; + } + } + } + { + if (true) { + throw (Error) jjte000; + } + } } finally { - if (jjtc000) { - jjtree.closeNodeScope(jjtn000, true); + if (jjtc000) { + jjtree.closeNodeScope(jjtn000, true); + } } - } throw new Error("Missing return statement in function"); } final public SimpleNode ConstMapContents() throws ParseException { - /*@bgen(jjtree) ConstMapContents */ - DynamicSerDeConstMapContents jjtn000 = new DynamicSerDeConstMapContents(JJTCONSTMAPCONTENTS); - boolean jjtc000 = true; - jjtree.openNodeScope(jjtn000); + /* @bgen(jjtree) ConstMapContents */ + DynamicSerDeConstMapContents jjtn000 = new DynamicSerDeConstMapContents( + JJTCONSTMAPCONTENTS); + boolean jjtc000 = true; + jjtree.openNodeScope(jjtn000); try { - switch ((jj_ntk==-1)?jj_ntk():jj_ntk) { + switch ((jj_ntk == -1) ? jj_ntk() : jj_ntk) { case tok_int_constant: case tok_double_constant: case IDENTIFIER: case tok_literal: case 61: case 64: - label_6: - while (true) { + label_6: while (true) { ConstValue(); jj_consume_token(66); ConstValue(); - switch ((jj_ntk==-1)?jj_ntk():jj_ntk) { + switch ((jj_ntk == -1) ? jj_ntk() : jj_ntk) { case 59: case 60: CommaOrSemicolon(); @@ -1113,7 +1483,7 @@ jj_la1[17] = jj_gen; ; } - switch ((jj_ntk==-1)?jj_ntk():jj_ntk) { + switch ((jj_ntk == -1) ? jj_ntk() : jj_ntk) { case tok_int_constant: case tok_double_constant: case IDENTIFIER: @@ -1127,43 +1497,59 @@ break label_6; } } - jjtree.closeNodeScope(jjtn000, true); - jjtc000 = false; + jjtree.closeNodeScope(jjtn000, true); + jjtc000 = false; break; default: jj_la1[19] = jj_gen; - jjtree.closeNodeScope(jjtn000, true); - jjtc000 = false; - {if (true) return jjtn000;} + jjtree.closeNodeScope(jjtn000, true); + jjtc000 = false; + { + if (true) { + return jjtn000; + } + } } } catch (Throwable jjte000) { - if (jjtc000) { - jjtree.clearNodeScope(jjtn000); - jjtc000 = false; - } else { - jjtree.popNode(); - } - if (jjte000 instanceof RuntimeException) { - {if (true) throw (RuntimeException)jjte000;} - } - if (jjte000 instanceof ParseException) { - {if (true) throw (ParseException)jjte000;} - } - {if (true) throw (Error)jjte000;} + if (jjtc000) { + jjtree.clearNodeScope(jjtn000); + jjtc000 = false; + } else { + jjtree.popNode(); + } + if (jjte000 instanceof RuntimeException) { + { + if (true) { + throw (RuntimeException) jjte000; + } + } + } + if (jjte000 instanceof ParseException) { + { + if (true) { + throw (ParseException) jjte000; + } + } + } + { + if (true) { + throw (Error) jjte000; + } + } } finally { - if (jjtc000) { - jjtree.closeNodeScope(jjtn000, true); + if (jjtc000) { + jjtree.closeNodeScope(jjtn000, true); + } } - } throw new Error("Missing return statement in function"); } final public DynamicSerDeStruct Struct() throws ParseException { - /*@bgen(jjtree) Struct */ - DynamicSerDeStruct jjtn000 = new DynamicSerDeStruct(JJTSTRUCT); - boolean jjtc000 = true; - jjtree.openNodeScope(jjtn000); + /* @bgen(jjtree) Struct */ + DynamicSerDeStruct jjtn000 = new DynamicSerDeStruct(JJTSTRUCT); + boolean jjtc000 = true; + jjtree.openNodeScope(jjtn000); try { jj_consume_token(tok_struct); jjtn000.name = jj_consume_token(IDENTIFIER).image; @@ -1172,35 +1558,51 @@ jj_consume_token(62); jjtree.closeNodeScope(jjtn000, true); jjtc000 = false; - this.types.put(jjtn000.name,jjtn000); - {if (true) return jjtn000;} + types.put(jjtn000.name, jjtn000); + { + if (true) { + return jjtn000; + } + } } catch (Throwable jjte000) { - if (jjtc000) { - jjtree.clearNodeScope(jjtn000); - jjtc000 = false; - } else { - jjtree.popNode(); - } - if (jjte000 instanceof RuntimeException) { - {if (true) throw (RuntimeException)jjte000;} - } - if (jjte000 instanceof ParseException) { - {if (true) throw (ParseException)jjte000;} - } - {if (true) throw (Error)jjte000;} + if (jjtc000) { + jjtree.clearNodeScope(jjtn000); + jjtc000 = false; + } else { + jjtree.popNode(); + } + if (jjte000 instanceof RuntimeException) { + { + if (true) { + throw (RuntimeException) jjte000; + } + } + } + if (jjte000 instanceof ParseException) { + { + if (true) { + throw (ParseException) jjte000; + } + } + } + { + if (true) { + throw (Error) jjte000; + } + } } finally { - if (jjtc000) { - jjtree.closeNodeScope(jjtn000, true); + if (jjtc000) { + jjtree.closeNodeScope(jjtn000, true); + } } - } throw new Error("Missing return statement in function"); } final public SimpleNode Xception() throws ParseException { - /*@bgen(jjtree) Xception */ - DynamicSerDeXception jjtn000 = new DynamicSerDeXception(JJTXCEPTION); - boolean jjtc000 = true; - jjtree.openNodeScope(jjtn000); + /* @bgen(jjtree) Xception */ + DynamicSerDeXception jjtn000 = new DynamicSerDeXception(JJTXCEPTION); + boolean jjtc000 = true; + jjtree.openNodeScope(jjtn000); try { jj_consume_token(tok_exception); jj_consume_token(IDENTIFIER); @@ -1209,44 +1611,59 @@ jj_consume_token(62); jjtree.closeNodeScope(jjtn000, true); jjtc000 = false; - {if (true) return jjtn000;} + { + if (true) { + return jjtn000; + } + } } catch (Throwable jjte000) { - if (jjtc000) { - jjtree.clearNodeScope(jjtn000); - jjtc000 = false; - } else { - jjtree.popNode(); - } - if (jjte000 instanceof RuntimeException) { - {if (true) throw (RuntimeException)jjte000;} - } - if (jjte000 instanceof ParseException) { - {if (true) throw (ParseException)jjte000;} - } - {if (true) throw (Error)jjte000;} + if (jjtc000) { + jjtree.clearNodeScope(jjtn000); + jjtc000 = false; + } else { + jjtree.popNode(); + } + if (jjte000 instanceof RuntimeException) { + { + if (true) { + throw (RuntimeException) jjte000; + } + } + } + if (jjte000 instanceof ParseException) { + { + if (true) { + throw (ParseException) jjte000; + } + } + } + { + if (true) { + throw (Error) jjte000; + } + } } finally { - if (jjtc000) { - jjtree.closeNodeScope(jjtn000, true); + if (jjtc000) { + jjtree.closeNodeScope(jjtn000, true); + } } - } throw new Error("Missing return statement in function"); } final public SimpleNode Service() throws ParseException { - /*@bgen(jjtree) Service */ - DynamicSerDeService jjtn000 = new DynamicSerDeService(JJTSERVICE); - boolean jjtc000 = true; - jjtree.openNodeScope(jjtn000); + /* @bgen(jjtree) Service */ + DynamicSerDeService jjtn000 = new DynamicSerDeService(JJTSERVICE); + boolean jjtc000 = true; + jjtree.openNodeScope(jjtn000); try { jj_consume_token(tok_service); jj_consume_token(IDENTIFIER); Extends(); jj_consume_token(61); FlagArgs(); - label_7: - while (true) { + label_7: while (true) { Function(); - switch ((jj_ntk==-1)?jj_ntk():jj_ntk) { + switch ((jj_ntk == -1) ? jj_ntk() : jj_ntk) { case tok_void: case tok_bool: case tok_i16: @@ -1270,39 +1687,59 @@ jj_consume_token(62); jjtree.closeNodeScope(jjtn000, true); jjtc000 = false; - // at some point, these should be inserted as a "db" - {if (true) return jjtn000;} + // at some point, these should be inserted as a "db" + { + if (true) { + return jjtn000; + } + } } catch (Throwable jjte000) { - if (jjtc000) { - jjtree.clearNodeScope(jjtn000); - jjtc000 = false; - } else { - jjtree.popNode(); - } - if (jjte000 instanceof RuntimeException) { - {if (true) throw (RuntimeException)jjte000;} - } - if (jjte000 instanceof ParseException) { - {if (true) throw (ParseException)jjte000;} - } - {if (true) throw (Error)jjte000;} + if (jjtc000) { + jjtree.clearNodeScope(jjtn000); + jjtc000 = false; + } else { + jjtree.popNode(); + } + if (jjte000 instanceof RuntimeException) { + { + if (true) { + throw (RuntimeException) jjte000; + } + } + } + if (jjte000 instanceof ParseException) { + { + if (true) { + throw (ParseException) jjte000; + } + } + } + { + if (true) { + throw (Error) jjte000; + } + } } finally { - if (jjtc000) { - jjtree.closeNodeScope(jjtn000, true); + if (jjtc000) { + jjtree.closeNodeScope(jjtn000, true); + } } - } throw new Error("Missing return statement in function"); } final public SimpleNode FlagArgs() throws ParseException { - /*@bgen(jjtree) FlagArgs */ - DynamicSerDeFlagArgs jjtn000 = new DynamicSerDeFlagArgs(JJTFLAGARGS); - boolean jjtc000 = true; - jjtree.openNodeScope(jjtn000); + /* @bgen(jjtree) FlagArgs */ + DynamicSerDeFlagArgs jjtn000 = new DynamicSerDeFlagArgs(JJTFLAGARGS); + boolean jjtc000 = true; + jjtree.openNodeScope(jjtn000); try { jjtree.closeNodeScope(jjtn000, true); jjtc000 = false; - {if (true) return jjtn000;} + { + if (true) { + return jjtn000; + } + } } finally { if (jjtc000) { jjtree.closeNodeScope(jjtn000, true); @@ -1312,14 +1749,18 @@ } final public SimpleNode UnflagArgs() throws ParseException { - /*@bgen(jjtree) UnflagArgs */ - DynamicSerDeUnflagArgs jjtn000 = new DynamicSerDeUnflagArgs(JJTUNFLAGARGS); - boolean jjtc000 = true; - jjtree.openNodeScope(jjtn000); + /* @bgen(jjtree) UnflagArgs */ + DynamicSerDeUnflagArgs jjtn000 = new DynamicSerDeUnflagArgs(JJTUNFLAGARGS); + boolean jjtc000 = true; + jjtree.openNodeScope(jjtn000); try { jjtree.closeNodeScope(jjtn000, true); jjtc000 = false; - {if (true) return jjtn000;} + { + if (true) { + return jjtn000; + } + } } finally { if (jjtc000) { jjtree.closeNodeScope(jjtn000, true); @@ -1329,48 +1770,56 @@ } final public SimpleNode Extends() throws ParseException { - /*@bgen(jjtree) Extends */ - DynamicSerDeExtends jjtn000 = new DynamicSerDeExtends(JJTEXTENDS); - boolean jjtc000 = true; - jjtree.openNodeScope(jjtn000); + /* @bgen(jjtree) Extends */ + DynamicSerDeExtends jjtn000 = new DynamicSerDeExtends(JJTEXTENDS); + boolean jjtc000 = true; + jjtree.openNodeScope(jjtn000); try { - switch ((jj_ntk==-1)?jj_ntk():jj_ntk) { + switch ((jj_ntk == -1) ? jj_ntk() : jj_ntk) { case tok_extends: jj_consume_token(tok_extends); jj_consume_token(IDENTIFIER); - jjtree.closeNodeScope(jjtn000, true); - jjtc000 = false; - {if (true) return jjtn000;} + jjtree.closeNodeScope(jjtn000, true); + jjtc000 = false; + { + if (true) { + return jjtn000; + } + } break; default: jj_la1[21] = jj_gen; - jjtree.closeNodeScope(jjtn000, true); - jjtc000 = false; - {if (true) return jjtn000;} + jjtree.closeNodeScope(jjtn000, true); + jjtc000 = false; + { + if (true) { + return jjtn000; + } + } } } finally { - if (jjtc000) { - jjtree.closeNodeScope(jjtn000, true); + if (jjtc000) { + jjtree.closeNodeScope(jjtn000, true); + } } - } throw new Error("Missing return statement in function"); } final public DynamicSerDeFunction Function() throws ParseException { - /*@bgen(jjtree) Function */ - DynamicSerDeFunction jjtn000 = new DynamicSerDeFunction(JJTFUNCTION); - boolean jjtc000 = true; - jjtree.openNodeScope(jjtn000); + /* @bgen(jjtree) Function */ + DynamicSerDeFunction jjtn000 = new DynamicSerDeFunction(JJTFUNCTION); + boolean jjtc000 = true; + jjtree.openNodeScope(jjtn000); try { Async(); FunctionType(); // the name of the function/table - jjtn000.name = jj_consume_token(IDENTIFIER).image; + jjtn000.name = jj_consume_token(IDENTIFIER).image; jj_consume_token(67); FieldList(); jj_consume_token(68); Throws(); - switch ((jj_ntk==-1)?jj_ntk():jj_ntk) { + switch ((jj_ntk == -1) ? jj_ntk() : jj_ntk) { case 59: case 60: CommaOrSemicolon(); @@ -1381,60 +1830,76 @@ } jjtree.closeNodeScope(jjtn000, true); jjtc000 = false; - this.tables.put(jjtn000.name, jjtn000); - {if (true) return jjtn000;} + tables.put(jjtn000.name, jjtn000); + { + if (true) { + return jjtn000; + } + } } catch (Throwable jjte000) { - if (jjtc000) { - jjtree.clearNodeScope(jjtn000); - jjtc000 = false; - } else { - jjtree.popNode(); - } - if (jjte000 instanceof RuntimeException) { - {if (true) throw (RuntimeException)jjte000;} - } - if (jjte000 instanceof ParseException) { - {if (true) throw (ParseException)jjte000;} - } - {if (true) throw (Error)jjte000;} + if (jjtc000) { + jjtree.clearNodeScope(jjtn000); + jjtc000 = false; + } else { + jjtree.popNode(); + } + if (jjte000 instanceof RuntimeException) { + { + if (true) { + throw (RuntimeException) jjte000; + } + } + } + if (jjte000 instanceof ParseException) { + { + if (true) { + throw (ParseException) jjte000; + } + } + } + { + if (true) { + throw (Error) jjte000; + } + } } finally { - if (jjtc000) { - jjtree.closeNodeScope(jjtn000, true); + if (jjtc000) { + jjtree.closeNodeScope(jjtn000, true); + } } - } throw new Error("Missing return statement in function"); } final public void Async() throws ParseException { - /*@bgen(jjtree) Async */ - DynamicSerDeAsync jjtn000 = new DynamicSerDeAsync(JJTASYNC); - boolean jjtc000 = true; - jjtree.openNodeScope(jjtn000); + /* @bgen(jjtree) Async */ + DynamicSerDeAsync jjtn000 = new DynamicSerDeAsync(JJTASYNC); + boolean jjtc000 = true; + jjtree.openNodeScope(jjtn000); try { - switch ((jj_ntk==-1)?jj_ntk():jj_ntk) { + switch ((jj_ntk == -1) ? jj_ntk() : jj_ntk) { case tok_async: jj_consume_token(tok_async); break; default: jj_la1[23] = jj_gen; - jjtree.closeNodeScope(jjtn000, true); - jjtc000 = false; + jjtree.closeNodeScope(jjtn000, true); + jjtc000 = false; } } finally { - if (jjtc000) { - jjtree.closeNodeScope(jjtn000, true); + if (jjtc000) { + jjtree.closeNodeScope(jjtn000, true); + } } - } } final public void Throws() throws ParseException { - /*@bgen(jjtree) Throws */ - DynamicSerDeThrows jjtn000 = new DynamicSerDeThrows(JJTTHROWS); - boolean jjtc000 = true; - jjtree.openNodeScope(jjtn000); + /* @bgen(jjtree) Throws */ + DynamicSerDeThrows jjtn000 = new DynamicSerDeThrows(JJTTHROWS); + boolean jjtc000 = true; + jjtree.openNodeScope(jjtn000); try { - switch ((jj_ntk==-1)?jj_ntk():jj_ntk) { + switch ((jj_ntk == -1) ? jj_ntk() : jj_ntk) { case tok_throws: jj_consume_token(tok_throws); jj_consume_token(67); @@ -1443,41 +1908,54 @@ break; default: jj_la1[24] = jj_gen; - jjtree.closeNodeScope(jjtn000, true); - jjtc000 = false; + jjtree.closeNodeScope(jjtn000, true); + jjtc000 = false; } } catch (Throwable jjte000) { - if (jjtc000) { - jjtree.clearNodeScope(jjtn000); - jjtc000 = false; - } else { - jjtree.popNode(); - } - if (jjte000 instanceof RuntimeException) { - {if (true) throw (RuntimeException)jjte000;} - } - if (jjte000 instanceof ParseException) { - {if (true) throw (ParseException)jjte000;} - } - {if (true) throw (Error)jjte000;} + if (jjtc000) { + jjtree.clearNodeScope(jjtn000); + jjtc000 = false; + } else { + jjtree.popNode(); + } + if (jjte000 instanceof RuntimeException) { + { + if (true) { + throw (RuntimeException) jjte000; + } + } + } + if (jjte000 instanceof ParseException) { + { + if (true) { + throw (ParseException) jjte000; + } + } + } + { + if (true) { + throw (Error) jjte000; + } + } } finally { - if (jjtc000) { - jjtree.closeNodeScope(jjtn000, true); + if (jjtc000) { + jjtree.closeNodeScope(jjtn000, true); + } } - } } -// nothing special - just use the DynamicSerDeFieldList's children methods to access the fields + // nothing special - just use the DynamicSerDeFieldList's children methods to + // access the fields final public DynamicSerDeFieldList FieldList() throws ParseException { - /*@bgen(jjtree) FieldList */ - DynamicSerDeFieldList jjtn000 = new DynamicSerDeFieldList(JJTFIELDLIST); - boolean jjtc000 = true; - jjtree.openNodeScope(jjtn000);this.field_val = -1; + /* @bgen(jjtree) FieldList */ + DynamicSerDeFieldList jjtn000 = new DynamicSerDeFieldList(JJTFIELDLIST); + boolean jjtc000 = true; + jjtree.openNodeScope(jjtn000); + field_val = -1; try { - label_8: - while (true) { - switch ((jj_ntk==-1)?jj_ntk():jj_ntk) { + label_8: while (true) { + switch ((jj_ntk == -1) ? jj_ntk() : jj_ntk) { case tok_bool: case tok_i16: case tok_i32: @@ -1500,39 +1978,55 @@ } Field(); } - jjtree.closeNodeScope(jjtn000, true); - jjtc000 = false; - {if (true) return jjtn000;} + jjtree.closeNodeScope(jjtn000, true); + jjtc000 = false; + { + if (true) { + return jjtn000; + } + } } catch (Throwable jjte000) { - if (jjtc000) { - jjtree.clearNodeScope(jjtn000); - jjtc000 = false; - } else { - jjtree.popNode(); - } - if (jjte000 instanceof RuntimeException) { - {if (true) throw (RuntimeException)jjte000;} - } - if (jjte000 instanceof ParseException) { - {if (true) throw (ParseException)jjte000;} - } - {if (true) throw (Error)jjte000;} + if (jjtc000) { + jjtree.clearNodeScope(jjtn000); + jjtc000 = false; + } else { + jjtree.popNode(); + } + if (jjte000 instanceof RuntimeException) { + { + if (true) { + throw (RuntimeException) jjte000; + } + } + } + if (jjte000 instanceof ParseException) { + { + if (true) { + throw (ParseException) jjte000; + } + } + } + { + if (true) { + throw (Error) jjte000; + } + } } finally { - if (jjtc000) { - jjtree.closeNodeScope(jjtn000, true); + if (jjtc000) { + jjtree.closeNodeScope(jjtn000, true); + } } - } throw new Error("Missing return statement in function"); } final public DynamicSerDeField Field() throws ParseException { - /*@bgen(jjtree) Field */ - DynamicSerDeField jjtn000 = new DynamicSerDeField(JJTFIELD); - boolean jjtc000 = true; - jjtree.openNodeScope(jjtn000);String fidnum = ""; - String fid; + /* @bgen(jjtree) Field */ + DynamicSerDeField jjtn000 = new DynamicSerDeField(JJTFIELD); + boolean jjtc000 = true; + jjtree.openNodeScope(jjtn000); + String fidnum = ""; try { - switch ((jj_ntk==-1)?jj_ntk():jj_ntk) { + switch ((jj_ntk == -1) ? jj_ntk() : jj_ntk) { case tok_int_constant: fidnum = jj_consume_token(tok_int_constant).image; jj_consume_token(66); @@ -1544,9 +2038,9 @@ FieldRequiredness(); FieldType(); // the name of the field - not optional - jjtn000.name = jj_consume_token(IDENTIFIER).image; + jjtn000.name = jj_consume_token(IDENTIFIER).image; FieldValue(); - switch ((jj_ntk==-1)?jj_ntk():jj_ntk) { + switch ((jj_ntk == -1) ? jj_ntk() : jj_ntk) { case 59: case 60: CommaOrSemicolon(); @@ -1557,179 +2051,270 @@ } jjtree.closeNodeScope(jjtn000, true); jjtc000 = false; - if(fidnum.length() > 0) { - int fidInt = Integer.valueOf(fidnum); - jjtn000.fieldid = fidInt; - } else { - jjtn000.fieldid = this.field_val--; - } - {if (true) return jjtn000;} + if (fidnum.length() > 0) { + int fidInt = Integer.valueOf(fidnum); + jjtn000.fieldid = fidInt; + } else { + jjtn000.fieldid = field_val--; + } + { + if (true) { + return jjtn000; + } + } } catch (Throwable jjte000) { - if (jjtc000) { - jjtree.clearNodeScope(jjtn000); - jjtc000 = false; - } else { - jjtree.popNode(); - } - if (jjte000 instanceof RuntimeException) { - {if (true) throw (RuntimeException)jjte000;} - } - if (jjte000 instanceof ParseException) { - {if (true) throw (ParseException)jjte000;} - } - {if (true) throw (Error)jjte000;} + if (jjtc000) { + jjtree.clearNodeScope(jjtn000); + jjtc000 = false; + } else { + jjtree.popNode(); + } + if (jjte000 instanceof RuntimeException) { + { + if (true) { + throw (RuntimeException) jjte000; + } + } + } + if (jjte000 instanceof ParseException) { + { + if (true) { + throw (ParseException) jjte000; + } + } + } + { + if (true) { + throw (Error) jjte000; + } + } } finally { - if (jjtc000) { - jjtree.closeNodeScope(jjtn000, true); + if (jjtc000) { + jjtree.closeNodeScope(jjtn000, true); + } } - } throw new Error("Missing return statement in function"); } - final public DynamicSerDeFieldRequiredness FieldRequiredness() throws ParseException { - /*@bgen(jjtree) FieldRequiredness */ - DynamicSerDeFieldRequiredness jjtn000 = new DynamicSerDeFieldRequiredness(JJTFIELDREQUIREDNESS); - boolean jjtc000 = true; - jjtree.openNodeScope(jjtn000); + final public DynamicSerDeFieldRequiredness FieldRequiredness() + throws ParseException { + /* @bgen(jjtree) FieldRequiredness */ + DynamicSerDeFieldRequiredness jjtn000 = new DynamicSerDeFieldRequiredness( + JJTFIELDREQUIREDNESS); + boolean jjtc000 = true; + jjtree.openNodeScope(jjtn000); try { - switch ((jj_ntk==-1)?jj_ntk():jj_ntk) { + switch ((jj_ntk == -1) ? jj_ntk() : jj_ntk) { case tok_required: jj_consume_token(tok_required); - jjtree.closeNodeScope(jjtn000, true); - jjtc000 = false; - jjtn000.requiredness = DynamicSerDeFieldRequiredness.RequirednessTypes.Required; - {if (true) return jjtn000;} + jjtree.closeNodeScope(jjtn000, true); + jjtc000 = false; + jjtn000.requiredness = DynamicSerDeFieldRequiredness.RequirednessTypes.Required; + { + if (true) { + return jjtn000; + } + } break; case tok_optional: jj_consume_token(tok_optional); - jjtree.closeNodeScope(jjtn000, true); - jjtc000 = false; - jjtn000.requiredness = DynamicSerDeFieldRequiredness.RequirednessTypes.Optional; - {if (true) return jjtn000;} + jjtree.closeNodeScope(jjtn000, true); + jjtc000 = false; + jjtn000.requiredness = DynamicSerDeFieldRequiredness.RequirednessTypes.Optional; + { + if (true) { + return jjtn000; + } + } break; case tok_skip: jj_consume_token(tok_skip); - jjtree.closeNodeScope(jjtn000, true); - jjtc000 = false; - jjtn000.requiredness = DynamicSerDeFieldRequiredness.RequirednessTypes.Skippable; - {if (true) return jjtn000;} + jjtree.closeNodeScope(jjtn000, true); + jjtc000 = false; + jjtn000.requiredness = DynamicSerDeFieldRequiredness.RequirednessTypes.Skippable; + { + if (true) { + return jjtn000; + } + } break; default: jj_la1[28] = jj_gen; - jjtree.closeNodeScope(jjtn000, true); - jjtc000 = false; - {if (true) return jjtn000;} + jjtree.closeNodeScope(jjtn000, true); + jjtc000 = false; + { + if (true) { + return jjtn000; + } + } } } finally { - if (jjtc000) { - jjtree.closeNodeScope(jjtn000, true); + if (jjtc000) { + jjtree.closeNodeScope(jjtn000, true); + } } - } throw new Error("Missing return statement in function"); } final public SimpleNode FieldValue() throws ParseException { - /*@bgen(jjtree) FieldValue */ - DynamicSerDeFieldValue jjtn000 = new DynamicSerDeFieldValue(JJTFIELDVALUE); - boolean jjtc000 = true; - jjtree.openNodeScope(jjtn000); + /* @bgen(jjtree) FieldValue */ + DynamicSerDeFieldValue jjtn000 = new DynamicSerDeFieldValue(JJTFIELDVALUE); + boolean jjtc000 = true; + jjtree.openNodeScope(jjtn000); try { - switch ((jj_ntk==-1)?jj_ntk():jj_ntk) { + switch ((jj_ntk == -1) ? jj_ntk() : jj_ntk) { case 63: jj_consume_token(63); ConstValue(); - jjtree.closeNodeScope(jjtn000, true); - jjtc000 = false; - {if (true) return jjtn000;} + jjtree.closeNodeScope(jjtn000, true); + jjtc000 = false; + { + if (true) { + return jjtn000; + } + } break; default: jj_la1[29] = jj_gen; - jjtree.closeNodeScope(jjtn000, true); - jjtc000 = false; - {if (true) return jjtn000;} + jjtree.closeNodeScope(jjtn000, true); + jjtc000 = false; + { + if (true) { + return jjtn000; + } + } } } catch (Throwable jjte000) { - if (jjtc000) { - jjtree.clearNodeScope(jjtn000); - jjtc000 = false; - } else { - jjtree.popNode(); - } - if (jjte000 instanceof RuntimeException) { - {if (true) throw (RuntimeException)jjte000;} - } - if (jjte000 instanceof ParseException) { - {if (true) throw (ParseException)jjte000;} - } - {if (true) throw (Error)jjte000;} + if (jjtc000) { + jjtree.clearNodeScope(jjtn000); + jjtc000 = false; + } else { + jjtree.popNode(); + } + if (jjte000 instanceof RuntimeException) { + { + if (true) { + throw (RuntimeException) jjte000; + } + } + } + if (jjte000 instanceof ParseException) { + { + if (true) { + throw (ParseException) jjte000; + } + } + } + { + if (true) { + throw (Error) jjte000; + } + } } finally { - if (jjtc000) { - jjtree.closeNodeScope(jjtn000, true); + if (jjtc000) { + jjtree.closeNodeScope(jjtn000, true); + } } - } throw new Error("Missing return statement in function"); } final public SimpleNode DefinitionType() throws ParseException { - /*@bgen(jjtree) DefinitionType */ - DynamicSerDeDefinitionType jjtn000 = new DynamicSerDeDefinitionType(JJTDEFINITIONTYPE); - boolean jjtc000 = true; - jjtree.openNodeScope(jjtn000); + /* @bgen(jjtree) DefinitionType */ + DynamicSerDeDefinitionType jjtn000 = new DynamicSerDeDefinitionType( + JJTDEFINITIONTYPE); + boolean jjtc000 = true; + jjtree.openNodeScope(jjtn000); try { - switch ((jj_ntk==-1)?jj_ntk():jj_ntk) { + switch ((jj_ntk == -1) ? jj_ntk() : jj_ntk) { case tok_string: TypeString(); - jjtree.closeNodeScope(jjtn000, true); - jjtc000 = false; - {if (true) return jjtn000;} + jjtree.closeNodeScope(jjtn000, true); + jjtc000 = false; + { + if (true) { + return jjtn000; + } + } break; case tok_bool: TypeBool(); - jjtree.closeNodeScope(jjtn000, true); - jjtc000 = false; - {if (true) return jjtn000;} + jjtree.closeNodeScope(jjtn000, true); + jjtc000 = false; + { + if (true) { + return jjtn000; + } + } break; case tok_i16: Typei16(); - jjtree.closeNodeScope(jjtn000, true); - jjtc000 = false; - {if (true) return jjtn000;} + jjtree.closeNodeScope(jjtn000, true); + jjtc000 = false; + { + if (true) { + return jjtn000; + } + } break; case tok_i32: Typei32(); - jjtree.closeNodeScope(jjtn000, true); - jjtc000 = false; - {if (true) return jjtn000;} + jjtree.closeNodeScope(jjtn000, true); + jjtc000 = false; + { + if (true) { + return jjtn000; + } + } break; case tok_i64: Typei64(); - jjtree.closeNodeScope(jjtn000, true); - jjtc000 = false; - {if (true) return jjtn000;} + jjtree.closeNodeScope(jjtn000, true); + jjtc000 = false; + { + if (true) { + return jjtn000; + } + } break; case tok_double: TypeDouble(); - jjtree.closeNodeScope(jjtn000, true); - jjtc000 = false; - {if (true) return jjtn000;} + jjtree.closeNodeScope(jjtn000, true); + jjtc000 = false; + { + if (true) { + return jjtn000; + } + } break; case tok_map: TypeMap(); - jjtree.closeNodeScope(jjtn000, true); - jjtc000 = false; - {if (true) return jjtn000;} + jjtree.closeNodeScope(jjtn000, true); + jjtc000 = false; + { + if (true) { + return jjtn000; + } + } break; case tok_set: TypeSet(); - jjtree.closeNodeScope(jjtn000, true); - jjtc000 = false; - {if (true) return jjtn000;} + jjtree.closeNodeScope(jjtn000, true); + jjtc000 = false; + { + if (true) { + return jjtn000; + } + } break; case tok_list: TypeList(); - jjtree.closeNodeScope(jjtn000, true); - jjtc000 = false; - {if (true) return jjtn000;} + jjtree.closeNodeScope(jjtn000, true); + jjtc000 = false; + { + if (true) { + return jjtn000; + } + } break; default: jj_la1[30] = jj_gen; @@ -1737,34 +2322,47 @@ throw new ParseException(); } } catch (Throwable jjte000) { - if (jjtc000) { - jjtree.clearNodeScope(jjtn000); - jjtc000 = false; - } else { - jjtree.popNode(); - } - if (jjte000 instanceof RuntimeException) { - {if (true) throw (RuntimeException)jjte000;} - } - if (jjte000 instanceof ParseException) { - {if (true) throw (ParseException)jjte000;} - } - {if (true) throw (Error)jjte000;} + if (jjtc000) { + jjtree.clearNodeScope(jjtn000); + jjtc000 = false; + } else { + jjtree.popNode(); + } + if (jjte000 instanceof RuntimeException) { + { + if (true) { + throw (RuntimeException) jjte000; + } + } + } + if (jjte000 instanceof ParseException) { + { + if (true) { + throw (ParseException) jjte000; + } + } + } + { + if (true) { + throw (Error) jjte000; + } + } } finally { - if (jjtc000) { - jjtree.closeNodeScope(jjtn000, true); + if (jjtc000) { + jjtree.closeNodeScope(jjtn000, true); + } } - } throw new Error("Missing return statement in function"); } final public void FunctionType() throws ParseException { - /*@bgen(jjtree) FunctionType */ - DynamicSerDeFunctionType jjtn000 = new DynamicSerDeFunctionType(JJTFUNCTIONTYPE); - boolean jjtc000 = true; - jjtree.openNodeScope(jjtn000); + /* @bgen(jjtree) FunctionType */ + DynamicSerDeFunctionType jjtn000 = new DynamicSerDeFunctionType( + JJTFUNCTIONTYPE); + boolean jjtc000 = true; + jjtree.openNodeScope(jjtn000); try { - switch ((jj_ntk==-1)?jj_ntk():jj_ntk) { + switch ((jj_ntk == -1) ? jj_ntk() : jj_ntk) { case tok_bool: case tok_i16: case tok_i32: @@ -1779,8 +2377,8 @@ break; case tok_void: jj_consume_token(tok_void); - jjtree.closeNodeScope(jjtn000, true); - jjtc000 = false; + jjtree.closeNodeScope(jjtn000, true); + jjtc000 = false; break; default: @@ -1789,98 +2387,157 @@ throw new ParseException(); } } catch (Throwable jjte000) { - if (jjtc000) { - jjtree.clearNodeScope(jjtn000); - jjtc000 = false; - } else { - jjtree.popNode(); - } - if (jjte000 instanceof RuntimeException) { - {if (true) throw (RuntimeException)jjte000;} - } - if (jjte000 instanceof ParseException) { - {if (true) throw (ParseException)jjte000;} - } - {if (true) throw (Error)jjte000;} + if (jjtc000) { + jjtree.clearNodeScope(jjtn000); + jjtc000 = false; + } else { + jjtree.popNode(); + } + if (jjte000 instanceof RuntimeException) { + { + if (true) { + throw (RuntimeException) jjte000; + } + } + } + if (jjte000 instanceof ParseException) { + { + if (true) { + throw (ParseException) jjte000; + } + } + } + { + if (true) { + throw (Error) jjte000; + } + } } finally { - if (jjtc000) { - jjtree.closeNodeScope(jjtn000, true); + if (jjtc000) { + jjtree.closeNodeScope(jjtn000, true); + } } - } } final public DynamicSerDeFieldType FieldType() throws ParseException { - /*@bgen(jjtree) FieldType */ - DynamicSerDeFieldType jjtn000 = new DynamicSerDeFieldType(JJTFIELDTYPE); - boolean jjtc000 = true; - jjtree.openNodeScope(jjtn000); + /* @bgen(jjtree) FieldType */ + DynamicSerDeFieldType jjtn000 = new DynamicSerDeFieldType(JJTFIELDTYPE); + boolean jjtc000 = true; + jjtree.openNodeScope(jjtn000); try { - switch ((jj_ntk==-1)?jj_ntk():jj_ntk) { + switch ((jj_ntk == -1) ? jj_ntk() : jj_ntk) { case tok_string: TypeString(); - jjtree.closeNodeScope(jjtn000, true); - jjtc000 = false; - {if (true) return jjtn000;} + jjtree.closeNodeScope(jjtn000, true); + jjtc000 = false; + { + if (true) { + return jjtn000; + } + } break; case tok_bool: TypeBool(); - jjtree.closeNodeScope(jjtn000, true); - jjtc000 = false; - {if (true) return jjtn000;} + jjtree.closeNodeScope(jjtn000, true); + jjtc000 = false; + { + if (true) { + return jjtn000; + } + } break; case tok_i16: Typei16(); - jjtree.closeNodeScope(jjtn000, true); - jjtc000 = false; - {if (true) return jjtn000;} + jjtree.closeNodeScope(jjtn000, true); + jjtc000 = false; + { + if (true) { + return jjtn000; + } + } break; case tok_i32: Typei32(); - jjtree.closeNodeScope(jjtn000, true); - jjtc000 = false; - {if (true) return jjtn000;} + jjtree.closeNodeScope(jjtn000, true); + jjtc000 = false; + { + if (true) { + return jjtn000; + } + } break; case tok_i64: Typei64(); - jjtree.closeNodeScope(jjtn000, true); - jjtc000 = false; - {if (true) return jjtn000;} + jjtree.closeNodeScope(jjtn000, true); + jjtc000 = false; + { + if (true) { + return jjtn000; + } + } break; case tok_double: TypeDouble(); - jjtree.closeNodeScope(jjtn000, true); - jjtc000 = false; - {if (true) return jjtn000;} + jjtree.closeNodeScope(jjtn000, true); + jjtc000 = false; + { + if (true) { + return jjtn000; + } + } break; case tok_map: TypeMap(); - jjtree.closeNodeScope(jjtn000, true); - jjtc000 = false; - {if (true) return jjtn000;} + jjtree.closeNodeScope(jjtn000, true); + jjtc000 = false; + { + if (true) { + return jjtn000; + } + } break; case tok_set: TypeSet(); - jjtree.closeNodeScope(jjtn000, true); - jjtc000 = false; - {if (true) return jjtn000;} + jjtree.closeNodeScope(jjtn000, true); + jjtc000 = false; + { + if (true) { + return jjtn000; + } + } break; case tok_list: TypeList(); - jjtree.closeNodeScope(jjtn000, true); - jjtc000 = false; - {if (true) return jjtn000;} + jjtree.closeNodeScope(jjtn000, true); + jjtc000 = false; + { + if (true) { + return jjtn000; + } + } break; case IDENTIFIER: jjtn000.name = jj_consume_token(IDENTIFIER).image; - jjtree.closeNodeScope(jjtn000, true); - jjtc000 = false; - if (this.types.get(jjtn000.name) == null) { - System.err.println("ERROR: DDL specifying type " + jjtn000.name + " which has not been defined"); - {if (true) throw new RuntimeException("specifying type " + jjtn000.name + " which has not been defined");} - } - // lookup the specified type and set this nodes type to it. Precludes forward and self references for now. - jjtn000.jjtAddChild(this.types.get(jjtn000.name),0); - {if (true) return jjtn000;} + jjtree.closeNodeScope(jjtn000, true); + jjtc000 = false; + if (types.get(jjtn000.name) == null) { + System.err.println("ERROR: DDL specifying type " + jjtn000.name + + " which has not been defined"); + { + if (true) { + throw new RuntimeException("specifying type " + jjtn000.name + + " which has not been defined"); + } + } + } + // lookup the specified type and set this nodes type to it. Precludes + // forward and self references for now. + jjtn000.jjtAddChild(types.get(jjtn000.name), 0); + { + if (true) { + return jjtn000; + } + } break; default: jj_la1[32] = jj_gen; @@ -1888,158 +2545,198 @@ throw new ParseException(); } } catch (Throwable jjte000) { - if (jjtc000) { - jjtree.clearNodeScope(jjtn000); - jjtc000 = false; - } else { - jjtree.popNode(); - } - if (jjte000 instanceof RuntimeException) { - {if (true) throw (RuntimeException)jjte000;} - } - if (jjte000 instanceof ParseException) { - {if (true) throw (ParseException)jjte000;} - } - {if (true) throw (Error)jjte000;} + if (jjtc000) { + jjtree.clearNodeScope(jjtn000); + jjtc000 = false; + } else { + jjtree.popNode(); + } + if (jjte000 instanceof RuntimeException) { + { + if (true) { + throw (RuntimeException) jjte000; + } + } + } + if (jjte000 instanceof ParseException) { + { + if (true) { + throw (ParseException) jjte000; + } + } + } + { + if (true) { + throw (Error) jjte000; + } + } } finally { - if (jjtc000) { - jjtree.closeNodeScope(jjtn000, true); + if (jjtc000) { + jjtree.closeNodeScope(jjtn000, true); + } } - } throw new Error("Missing return statement in function"); } final public DynamicSerDeTypeString TypeString() throws ParseException { - /*@bgen(jjtree) TypeString */ - DynamicSerDeTypeString jjtn000 = new DynamicSerDeTypeString(JJTTYPESTRING); - boolean jjtc000 = true; - jjtree.openNodeScope(jjtn000); + /* @bgen(jjtree) TypeString */ + DynamicSerDeTypeString jjtn000 = new DynamicSerDeTypeString(JJTTYPESTRING); + boolean jjtc000 = true; + jjtree.openNodeScope(jjtn000); try { jj_consume_token(tok_string); - jjtree.closeNodeScope(jjtn000, true); - jjtc000 = false; - {if (true) return jjtn000;} + jjtree.closeNodeScope(jjtn000, true); + jjtc000 = false; + { + if (true) { + return jjtn000; + } + } } finally { - if (jjtc000) { - jjtree.closeNodeScope(jjtn000, true); + if (jjtc000) { + jjtree.closeNodeScope(jjtn000, true); + } } - } throw new Error("Missing return statement in function"); } final public DynamicSerDeTypeByte TypeByte() throws ParseException { - /*@bgen(jjtree) TypeByte */ - DynamicSerDeTypeByte jjtn000 = new DynamicSerDeTypeByte(JJTTYPEBYTE); - boolean jjtc000 = true; - jjtree.openNodeScope(jjtn000); + /* @bgen(jjtree) TypeByte */ + DynamicSerDeTypeByte jjtn000 = new DynamicSerDeTypeByte(JJTTYPEBYTE); + boolean jjtc000 = true; + jjtree.openNodeScope(jjtn000); try { jj_consume_token(tok_byte); - jjtree.closeNodeScope(jjtn000, true); - jjtc000 = false; - {if (true) return jjtn000;} + jjtree.closeNodeScope(jjtn000, true); + jjtc000 = false; + { + if (true) { + return jjtn000; + } + } } finally { - if (jjtc000) { - jjtree.closeNodeScope(jjtn000, true); - } + if (jjtc000) { + jjtree.closeNodeScope(jjtn000, true); + } } throw new Error("Missing return statement in function"); } final public DynamicSerDeTypei16 Typei16() throws ParseException { - /*@bgen(jjtree) Typei16 */ - DynamicSerDeTypei16 jjtn000 = new DynamicSerDeTypei16(JJTTYPEI16); - boolean jjtc000 = true; - jjtree.openNodeScope(jjtn000); + /* @bgen(jjtree) Typei16 */ + DynamicSerDeTypei16 jjtn000 = new DynamicSerDeTypei16(JJTTYPEI16); + boolean jjtc000 = true; + jjtree.openNodeScope(jjtn000); try { jj_consume_token(tok_i16); - jjtree.closeNodeScope(jjtn000, true); - jjtc000 = false; - {if (true) return jjtn000;} + jjtree.closeNodeScope(jjtn000, true); + jjtc000 = false; + { + if (true) { + return jjtn000; + } + } } finally { - if (jjtc000) { - jjtree.closeNodeScope(jjtn000, true); + if (jjtc000) { + jjtree.closeNodeScope(jjtn000, true); + } } - } throw new Error("Missing return statement in function"); } final public DynamicSerDeTypei32 Typei32() throws ParseException { - /*@bgen(jjtree) Typei32 */ - DynamicSerDeTypei32 jjtn000 = new DynamicSerDeTypei32(JJTTYPEI32); - boolean jjtc000 = true; - jjtree.openNodeScope(jjtn000); + /* @bgen(jjtree) Typei32 */ + DynamicSerDeTypei32 jjtn000 = new DynamicSerDeTypei32(JJTTYPEI32); + boolean jjtc000 = true; + jjtree.openNodeScope(jjtn000); try { jj_consume_token(tok_i32); - jjtree.closeNodeScope(jjtn000, true); - jjtc000 = false; - {if (true) return jjtn000;} + jjtree.closeNodeScope(jjtn000, true); + jjtc000 = false; + { + if (true) { + return jjtn000; + } + } } finally { - if (jjtc000) { - jjtree.closeNodeScope(jjtn000, true); + if (jjtc000) { + jjtree.closeNodeScope(jjtn000, true); + } } - } throw new Error("Missing return statement in function"); } final public DynamicSerDeTypei64 Typei64() throws ParseException { - /*@bgen(jjtree) Typei64 */ - DynamicSerDeTypei64 jjtn000 = new DynamicSerDeTypei64(JJTTYPEI64); - boolean jjtc000 = true; - jjtree.openNodeScope(jjtn000); + /* @bgen(jjtree) Typei64 */ + DynamicSerDeTypei64 jjtn000 = new DynamicSerDeTypei64(JJTTYPEI64); + boolean jjtc000 = true; + jjtree.openNodeScope(jjtn000); try { jj_consume_token(tok_i64); - jjtree.closeNodeScope(jjtn000, true); - jjtc000 = false; - {if (true) return jjtn000;} + jjtree.closeNodeScope(jjtn000, true); + jjtc000 = false; + { + if (true) { + return jjtn000; + } + } } finally { - if (jjtc000) { - jjtree.closeNodeScope(jjtn000, true); + if (jjtc000) { + jjtree.closeNodeScope(jjtn000, true); + } } - } throw new Error("Missing return statement in function"); } final public DynamicSerDeTypeDouble TypeDouble() throws ParseException { - /*@bgen(jjtree) TypeDouble */ - DynamicSerDeTypeDouble jjtn000 = new DynamicSerDeTypeDouble(JJTTYPEDOUBLE); - boolean jjtc000 = true; - jjtree.openNodeScope(jjtn000); + /* @bgen(jjtree) TypeDouble */ + DynamicSerDeTypeDouble jjtn000 = new DynamicSerDeTypeDouble(JJTTYPEDOUBLE); + boolean jjtc000 = true; + jjtree.openNodeScope(jjtn000); try { jj_consume_token(tok_double); - jjtree.closeNodeScope(jjtn000, true); - jjtc000 = false; - {if (true) return jjtn000;} + jjtree.closeNodeScope(jjtn000, true); + jjtc000 = false; + { + if (true) { + return jjtn000; + } + } } finally { - if (jjtc000) { - jjtree.closeNodeScope(jjtn000, true); + if (jjtc000) { + jjtree.closeNodeScope(jjtn000, true); + } } - } throw new Error("Missing return statement in function"); } final public DynamicSerDeTypeBool TypeBool() throws ParseException { - /*@bgen(jjtree) TypeBool */ - DynamicSerDeTypeBool jjtn000 = new DynamicSerDeTypeBool(JJTTYPEBOOL); - boolean jjtc000 = true; - jjtree.openNodeScope(jjtn000); + /* @bgen(jjtree) TypeBool */ + DynamicSerDeTypeBool jjtn000 = new DynamicSerDeTypeBool(JJTTYPEBOOL); + boolean jjtc000 = true; + jjtree.openNodeScope(jjtn000); try { jj_consume_token(tok_bool); - jjtree.closeNodeScope(jjtn000, true); - jjtc000 = false; - {if (true) return jjtn000;} + jjtree.closeNodeScope(jjtn000, true); + jjtc000 = false; + { + if (true) { + return jjtn000; + } + } } finally { - if (jjtc000) { - jjtree.closeNodeScope(jjtn000, true); + if (jjtc000) { + jjtree.closeNodeScope(jjtn000, true); + } } - } throw new Error("Missing return statement in function"); } final public DynamicSerDeTypeMap TypeMap() throws ParseException { - /*@bgen(jjtree) TypeMap */ - DynamicSerDeTypeMap jjtn000 = new DynamicSerDeTypeMap(JJTTYPEMAP); - boolean jjtc000 = true; - jjtree.openNodeScope(jjtn000); + /* @bgen(jjtree) TypeMap */ + DynamicSerDeTypeMap jjtn000 = new DynamicSerDeTypeMap(JJTTYPEMAP); + boolean jjtc000 = true; + jjtree.openNodeScope(jjtn000); try { jj_consume_token(tok_map); jj_consume_token(69); @@ -2049,34 +2746,50 @@ jj_consume_token(70); jjtree.closeNodeScope(jjtn000, true); jjtc000 = false; - {if (true) return jjtn000;} + { + if (true) { + return jjtn000; + } + } } catch (Throwable jjte000) { - if (jjtc000) { - jjtree.clearNodeScope(jjtn000); - jjtc000 = false; - } else { - jjtree.popNode(); - } - if (jjte000 instanceof RuntimeException) { - {if (true) throw (RuntimeException)jjte000;} - } - if (jjte000 instanceof ParseException) { - {if (true) throw (ParseException)jjte000;} - } - {if (true) throw (Error)jjte000;} + if (jjtc000) { + jjtree.clearNodeScope(jjtn000); + jjtc000 = false; + } else { + jjtree.popNode(); + } + if (jjte000 instanceof RuntimeException) { + { + if (true) { + throw (RuntimeException) jjte000; + } + } + } + if (jjte000 instanceof ParseException) { + { + if (true) { + throw (ParseException) jjte000; + } + } + } + { + if (true) { + throw (Error) jjte000; + } + } } finally { - if (jjtc000) { - jjtree.closeNodeScope(jjtn000, true); + if (jjtc000) { + jjtree.closeNodeScope(jjtn000, true); + } } - } throw new Error("Missing return statement in function"); } final public DynamicSerDeTypeSet TypeSet() throws ParseException { - /*@bgen(jjtree) TypeSet */ - DynamicSerDeTypeSet jjtn000 = new DynamicSerDeTypeSet(JJTTYPESET); - boolean jjtc000 = true; - jjtree.openNodeScope(jjtn000); + /* @bgen(jjtree) TypeSet */ + DynamicSerDeTypeSet jjtn000 = new DynamicSerDeTypeSet(JJTTYPESET); + boolean jjtc000 = true; + jjtree.openNodeScope(jjtn000); try { jj_consume_token(tok_set); jj_consume_token(69); @@ -2084,34 +2797,50 @@ jj_consume_token(70); jjtree.closeNodeScope(jjtn000, true); jjtc000 = false; - {if (true) return jjtn000;} + { + if (true) { + return jjtn000; + } + } } catch (Throwable jjte000) { - if (jjtc000) { - jjtree.clearNodeScope(jjtn000); - jjtc000 = false; - } else { - jjtree.popNode(); - } - if (jjte000 instanceof RuntimeException) { - {if (true) throw (RuntimeException)jjte000;} - } - if (jjte000 instanceof ParseException) { - {if (true) throw (ParseException)jjte000;} - } - {if (true) throw (Error)jjte000;} + if (jjtc000) { + jjtree.clearNodeScope(jjtn000); + jjtc000 = false; + } else { + jjtree.popNode(); + } + if (jjte000 instanceof RuntimeException) { + { + if (true) { + throw (RuntimeException) jjte000; + } + } + } + if (jjte000 instanceof ParseException) { + { + if (true) { + throw (ParseException) jjte000; + } + } + } + { + if (true) { + throw (Error) jjte000; + } + } } finally { - if (jjtc000) { - jjtree.closeNodeScope(jjtn000, true); + if (jjtc000) { + jjtree.closeNodeScope(jjtn000, true); + } } - } throw new Error("Missing return statement in function"); } final public DynamicSerDeTypeList TypeList() throws ParseException { - /*@bgen(jjtree) TypeList */ - DynamicSerDeTypeList jjtn000 = new DynamicSerDeTypeList(JJTTYPELIST); - boolean jjtc000 = true; - jjtree.openNodeScope(jjtn000); + /* @bgen(jjtree) TypeList */ + DynamicSerDeTypeList jjtn000 = new DynamicSerDeTypeList(JJTTYPELIST); + boolean jjtc000 = true; + jjtree.openNodeScope(jjtn000); try { jj_consume_token(tok_list); jj_consume_token(69); @@ -2119,26 +2848,42 @@ jj_consume_token(70); jjtree.closeNodeScope(jjtn000, true); jjtc000 = false; - {if (true) return jjtn000;} + { + if (true) { + return jjtn000; + } + } } catch (Throwable jjte000) { - if (jjtc000) { - jjtree.clearNodeScope(jjtn000); - jjtc000 = false; - } else { - jjtree.popNode(); - } - if (jjte000 instanceof RuntimeException) { - {if (true) throw (RuntimeException)jjte000;} - } - if (jjte000 instanceof ParseException) { - {if (true) throw (ParseException)jjte000;} - } - {if (true) throw (Error)jjte000;} + if (jjtc000) { + jjtree.clearNodeScope(jjtn000); + jjtc000 = false; + } else { + jjtree.popNode(); + } + if (jjte000 instanceof RuntimeException) { + { + if (true) { + throw (RuntimeException) jjte000; + } + } + } + if (jjte000 instanceof ParseException) { + { + if (true) { + throw (ParseException) jjte000; + } + } + } + { + if (true) { + throw (Error) jjte000; + } + } } finally { - if (jjtc000) { - jjtree.closeNodeScope(jjtn000, true); + if (jjtc000) { + jjtree.closeNodeScope(jjtn000, true); + } } - } throw new Error("Missing return statement in function"); } @@ -2156,47 +2901,74 @@ static private int[] jj_la1_1; static private int[] jj_la1_2; static { - jj_la1_init_0(); - jj_la1_init_1(); - jj_la1_init_2(); - } - private static void jj_la1_init_0() { - jj_la1_0 = new int[] {0x0,0x100,0xa3fee00,0xa3fee00,0x23fee00,0x100,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0xb0000000,0x0,0x0,0x0,0x0,0xa0000000,0x0,0x0,0x0,0x0,0xa0000000,0xb0000000,0xa0000000,}; - } - private static void jj_la1_init_1() { - jj_la1_1 = new int[] {0x18000000,0x18019c20,0x0,0x0,0x0,0x19c20,0x11c20,0x18000000,0x400000,0x80000000,0x18000000,0x2000000,0x18000000,0x18000000,0x22700000,0x18000000,0x22700000,0x18000000,0x22700000,0x22700000,0x4003cf,0x2000,0x18000000,0x200,0x4000,0x5e01cf,0x100000,0x18000000,0xe0000,0x80000000,0x1cf,0x4001cf,0x4001cf,}; - } - private static void jj_la1_init_2() { - jj_la1_2 = new int[] {0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x1,0x0,0x1,0x0,0x1,0x1,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,}; - } + jj_la1_init_0(); + jj_la1_init_1(); + jj_la1_init_2(); + } + private static void jj_la1_init_0() { + jj_la1_0 = new int[] { 0x0, 0x100, 0xa3fee00, 0xa3fee00, 0x23fee00, 0x100, + 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, + 0xb0000000, 0x0, 0x0, 0x0, 0x0, 0xa0000000, 0x0, 0x0, 0x0, 0x0, + 0xa0000000, 0xb0000000, 0xa0000000, }; + } + + private static void jj_la1_init_1() { + jj_la1_1 = new int[] { 0x18000000, 0x18019c20, 0x0, 0x0, 0x0, 0x19c20, + 0x11c20, 0x18000000, 0x400000, 0x80000000, 0x18000000, 0x2000000, + 0x18000000, 0x18000000, 0x22700000, 0x18000000, 0x22700000, 0x18000000, + 0x22700000, 0x22700000, 0x4003cf, 0x2000, 0x18000000, 0x200, 0x4000, + 0x5e01cf, 0x100000, 0x18000000, 0xe0000, 0x80000000, 0x1cf, 0x4001cf, + 0x4001cf, }; + } + + private static void jj_la1_init_2() { + jj_la1_2 = new int[] { 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, + 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x1, 0x0, 0x1, 0x1, 0x0, 0x0, 0x0, 0x0, + 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, }; + } + /** Constructor with InputStream. */ public thrift_grammar(java.io.InputStream stream) { - this(stream, null); + this(stream, null); } + /** Constructor with InputStream and supplied encoding */ public thrift_grammar(java.io.InputStream stream, String encoding) { - try { jj_input_stream = new SimpleCharStream(stream, encoding, 1, 1); } catch(java.io.UnsupportedEncodingException e) { throw new RuntimeException(e); } + try { + jj_input_stream = new SimpleCharStream(stream, encoding, 1, 1); + } catch (java.io.UnsupportedEncodingException e) { + throw new RuntimeException(e); + } token_source = new thrift_grammarTokenManager(jj_input_stream); token = new Token(); jj_ntk = -1; jj_gen = 0; - for (int i = 0; i < 33; i++) jj_la1[i] = -1; + for (int i = 0; i < 33; i++) { + jj_la1[i] = -1; + } } /** Reinitialise. */ public void ReInit(java.io.InputStream stream) { - ReInit(stream, null); + ReInit(stream, null); } + /** Reinitialise. */ public void ReInit(java.io.InputStream stream, String encoding) { - try { jj_input_stream.ReInit(stream, encoding, 1, 1); } catch(java.io.UnsupportedEncodingException e) { throw new RuntimeException(e); } + try { + jj_input_stream.ReInit(stream, encoding, 1, 1); + } catch (java.io.UnsupportedEncodingException e) { + throw new RuntimeException(e); + } token_source.ReInit(jj_input_stream); token = new Token(); jj_ntk = -1; jjtree.reset(); jj_gen = 0; - for (int i = 0; i < 33; i++) jj_la1[i] = -1; + for (int i = 0; i < 33; i++) { + jj_la1[i] = -1; + } } /** Constructor. */ @@ -2206,7 +2978,9 @@ token = new Token(); jj_ntk = -1; jj_gen = 0; - for (int i = 0; i < 33; i++) jj_la1[i] = -1; + for (int i = 0; i < 33; i++) { + jj_la1[i] = -1; + } } /** Reinitialise. */ @@ -2217,7 +2991,9 @@ jj_ntk = -1; jjtree.reset(); jj_gen = 0; - for (int i = 0; i < 33; i++) jj_la1[i] = -1; + for (int i = 0; i < 33; i++) { + jj_la1[i] = -1; + } } /** Constructor with generated Token Manager. */ @@ -2226,7 +3002,9 @@ token = new Token(); jj_ntk = -1; jj_gen = 0; - for (int i = 0; i < 33; i++) jj_la1[i] = -1; + for (int i = 0; i < 33; i++) { + jj_la1[i] = -1; + } } /** Reinitialise. */ @@ -2236,13 +3014,18 @@ jj_ntk = -1; jjtree.reset(); jj_gen = 0; - for (int i = 0; i < 33; i++) jj_la1[i] = -1; + for (int i = 0; i < 33; i++) { + jj_la1[i] = -1; + } } private Token jj_consume_token(int kind) throws ParseException { Token oldToken; - if ((oldToken = token).next != null) token = token.next; - else token = token.next = token_source.getNextToken(); + if ((oldToken = token).next != null) { + token = token.next; + } else { + token = token.next = token_source.getNextToken(); + } jj_ntk = -1; if (token.kind == kind) { jj_gen++; @@ -2253,34 +3036,40 @@ throw generateParseException(); } - -/** Get the next Token. */ + /** Get the next Token. */ final public Token getNextToken() { - if (token.next != null) token = token.next; - else token = token.next = token_source.getNextToken(); + if (token.next != null) { + token = token.next; + } else { + token = token.next = token_source.getNextToken(); + } jj_ntk = -1; jj_gen++; return token; } -/** Get the specific Token. */ + /** Get the specific Token. */ final public Token getToken(int index) { Token t = token; for (int i = 0; i < index; i++) { - if (t.next != null) t = t.next; - else t = t.next = token_source.getNextToken(); + if (t.next != null) { + t = t.next; + } else { + t = t.next = token_source.getNextToken(); + } } return t; } private int jj_ntk() { - if ((jj_nt=token.next) == null) - return (jj_ntk = (token.next=token_source.getNextToken()).kind); - else + if ((jj_nt = token.next) == null) { + return (jj_ntk = (token.next = token_source.getNextToken()).kind); + } else { return (jj_ntk = jj_nt.kind); + } } - private java.util.List jj_expentries = new java.util.ArrayList(); + private final java.util.List jj_expentries = new java.util.ArrayList(); private int[] jj_expentry; private int jj_kind = -1; @@ -2295,14 +3084,14 @@ for (int i = 0; i < 33; i++) { if (jj_la1[i] == jj_gen) { for (int j = 0; j < 32; j++) { - if ((jj_la1_0[i] & (1< + @Override public Class getRealType() { try { - Class c = this.getElementType().getRealType(); + Class c = getElementType().getRealType(); Object o = c.newInstance(); Set l = Collections.singleton(o); return l.getClass(); @@ -59,62 +59,68 @@ } public DynamicSerDeTypeBase getElementType() { - return (DynamicSerDeTypeBase)((DynamicSerDeFieldType)this.jjtGetChild(FD_TYPE)).getMyType(); + return ((DynamicSerDeFieldType) jjtGetChild(FD_TYPE)).getMyType(); } + @Override public String toString() { - return "set<" + this.getElementType().toString() + ">"; + return "set<" + getElementType().toString() + ">"; } + @Override public byte getType() { return TType.SET; } - /** NOTE: Set is not supported by Hive yet. + /** + * NOTE: Set is not supported by Hive yet. */ @Override public Object deserialize(Object reuse, TProtocol iprot) - throws SerDeException, TException, IllegalAccessException { + throws SerDeException, TException, IllegalAccessException { TSet theset = iprot.readSetBegin(); if (theset == null) { return null; } Set result; if (reuse != null) { - result = (Set)reuse; + result = (Set) reuse; result.clear(); } else { result = new HashSet(); } - for(int i = 0; i < theset.size; i++) { - Object elem = this.getElementType().deserialize(null, iprot); + for (int i = 0; i < theset.size; i++) { + Object elem = getElementType().deserialize(null, iprot); result.add(elem); } - // in theory, the below call isn't needed in non thrift_mode, but let's not get too crazy + // in theory, the below call isn't needed in non thrift_mode, but let's not + // get too crazy iprot.readSetEnd(); return result; } - /** NOTE: Set is not supported by Hive yet. - * The code uses ListObjectInspector right now. We need to change it to - * SetObjectInspector when that is done. + /** + * NOTE: Set is not supported by Hive yet. The code uses ListObjectInspector + * right now. We need to change it to SetObjectInspector when that is done. */ TSet tset = null; + @Override public void serialize(Object o, ObjectInspector oi, TProtocol oprot) - throws TException, SerDeException, NoSuchFieldException, - IllegalAccessException { + throws TException, SerDeException, NoSuchFieldException, + IllegalAccessException { - ListObjectInspector loi = (ListObjectInspector)oi; + ListObjectInspector loi = (ListObjectInspector) oi; - Set set = (Set)o; - DynamicSerDeTypeBase mt = this.getElementType(); + Set set = (Set) o; + DynamicSerDeTypeBase mt = getElementType(); tset = new TSet(mt.getType(), set.size()); oprot.writeSetBegin(tset); - for(Object element: set) { + for (Object element : set) { mt.serialize(element, loi.getListElementObjectInspector(), oprot); } - // in theory, the below call isn't needed in non thrift_mode, but let's not get too crazy + // in theory, the below call isn't needed in non thrift_mode, but let's not + // get too crazy oprot.writeSetEnd(); } } Index: serde/src/java/org/apache/hadoop/hive/serde2/ByteStreamTypedSerDe.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/ByteStreamTypedSerDe.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/ByteStreamTypedSerDe.java (working copy) @@ -21,7 +21,6 @@ import java.lang.reflect.Type; import org.apache.hadoop.io.BytesWritable; -import org.apache.hadoop.io.Text; import org.apache.hadoop.io.Writable; public abstract class ByteStreamTypedSerDe extends TypedSerDe { @@ -35,12 +34,12 @@ bis = new ByteStream.Input(); } + @Override public Object deserialize(Writable field) throws SerDeException { Object retObj = super.deserialize(field); - BytesWritable b = (BytesWritable)field; + BytesWritable b = (BytesWritable) field; bis.reset(b.get(), b.getSize()); return (retObj); } - } Index: serde/src/java/org/apache/hadoop/hive/serde2/TypedSerDe.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/TypedSerDe.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/TypedSerDe.java (working copy) @@ -37,20 +37,22 @@ public TypedSerDe(Type objectType) throws SerDeException { this.objectType = objectType; if (objectType instanceof Class) { - objectClass = (Class)objectType; + objectClass = (Class) objectType; } else if (objectType instanceof ParameterizedType) { - objectClass = (Class)(((ParameterizedType)objectType).getRawType()); + objectClass = (Class) (((ParameterizedType) objectType).getRawType()); } else { - throw new SerDeException("Cannot create TypedSerDe with type " + objectType); + throw new SerDeException("Cannot create TypedSerDe with type " + + objectType); } } protected Object deserializeCache; + public Object deserialize(Writable blob) throws SerDeException { if (deserializeCache == null) { return ReflectionUtils.newInstance(objectClass, null); } else { - assert(deserializeCache.getClass().equals(objectClass)); + assert (deserializeCache.getClass().equals(objectClass)); return deserializeCache; } } @@ -63,7 +65,7 @@ protected ObjectInspectorFactory.ObjectInspectorOptions getObjectInspectorOptions() { return ObjectInspectorFactory.ObjectInspectorOptions.JAVA; } - + public void initialize(Configuration job, Properties tbl) throws SerDeException { // do nothing @@ -72,7 +74,9 @@ public Class getSerializedClass() { return BytesWritable.class; } - public Writable serialize(Object obj, ObjectInspector objInspector) throws SerDeException { + + public Writable serialize(Object obj, ObjectInspector objInspector) + throws SerDeException { throw new RuntimeException("not supported"); } Index: serde/src/java/org/apache/hadoop/hive/serde2/ColumnSet.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/ColumnSet.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/ColumnSet.java (working copy) @@ -26,15 +26,14 @@ public ColumnSet() { } - public ColumnSet(ArrayList col) - { + public ColumnSet(ArrayList col) { this(); this.col = col; } + @Override public String toString() { return col.toString(); } - -} +} Index: serde/src/java/org/apache/hadoop/hive/serde2/Serializer.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/Serializer.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/Serializer.java (working copy) @@ -18,40 +18,46 @@ package org.apache.hadoop.hive.serde2; +import java.util.Properties; + +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.io.Writable; -import org.apache.hadoop.conf.Configuration; -import java.util.Properties; /** - * HiveSerializer is used to serialize data to a Hadoop Writable object. - * The serialize - * In addition to the interface below, all implementations are assume to have a ctor - * that takes a single 'Table' object as argument. - * + * HiveSerializer is used to serialize data to a Hadoop Writable object. The + * serialize In addition to the interface below, all implementations are assume + * to have a ctor that takes a single 'Table' object as argument. + * */ public interface Serializer { /** * Initialize the HiveSerializer. - * @param conf System properties - * @param tbl table properties + * + * @param conf + * System properties + * @param tbl + * table properties * @throws SerDeException */ - public void initialize(Configuration conf, Properties tbl) throws SerDeException; - + public void initialize(Configuration conf, Properties tbl) + throws SerDeException; + /** * Returns the Writable class that would be returned by the serialize method. * This is used to initialize SequenceFile header. */ public Class getSerializedClass(); + /** - * Serialize an object by navigating inside the Object with the ObjectInspector. - * In most cases, the return value of this function will be constant since the function - * will reuse the Writable object. - * If the client wants to keep a copy of the Writable, the client needs to clone the + * Serialize an object by navigating inside the Object with the + * ObjectInspector. In most cases, the return value of this function will be + * constant since the function will reuse the Writable object. If the client + * wants to keep a copy of the Writable, the client needs to clone the * returned value. */ - public Writable serialize(Object obj, ObjectInspector objInspector) throws SerDeException; + public Writable serialize(Object obj, ObjectInspector objInspector) + throws SerDeException; } Index: serde/src/java/org/apache/hadoop/hive/serde2/ColumnProjectionUtils.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/ColumnProjectionUtils.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/ColumnProjectionUtils.java (working copy) @@ -25,83 +25,86 @@ public class ColumnProjectionUtils { - public static String READ_COLUMN_IDS_CONF_STR = "hive.io.file.readcolumn.ids"; + public static String READ_COLUMN_IDS_CONF_STR = "hive.io.file.readcolumn.ids"; - /** - * Sets read columns' ids(start from zero) for RCFile's Reader. Once a column - * is included in the list, RCFile's reader will not skip its value. - * - */ - public static void setReadColumnIDs(Configuration conf, ArrayList ids) { - String id = toReadColumnIDString(ids); - setReadColumnIDConf(conf, id); - } + /** + * Sets read columns' ids(start from zero) for RCFile's Reader. Once a column + * is included in the list, RCFile's reader will not skip its value. + * + */ + public static void setReadColumnIDs(Configuration conf, ArrayList ids) { + String id = toReadColumnIDString(ids); + setReadColumnIDConf(conf, id); + } - /** - * Sets read columns' ids(start from zero) for RCFile's Reader. Once a column - * is included in the list, RCFile's reader will not skip its value. - * - */ - public static void appendReadColumnIDs(Configuration conf, - ArrayList ids) { - String id = toReadColumnIDString(ids); - if(id != null) { - String old = conf.get(READ_COLUMN_IDS_CONF_STR, null); - String newConfStr = id; - if (old != null) - newConfStr = newConfStr + StringUtils.COMMA_STR + old; + /** + * Sets read columns' ids(start from zero) for RCFile's Reader. Once a column + * is included in the list, RCFile's reader will not skip its value. + * + */ + public static void appendReadColumnIDs(Configuration conf, + ArrayList ids) { + String id = toReadColumnIDString(ids); + if (id != null) { + String old = conf.get(READ_COLUMN_IDS_CONF_STR, null); + String newConfStr = id; + if (old != null) { + newConfStr = newConfStr + StringUtils.COMMA_STR + old; + } - setReadColumnIDConf(conf, newConfStr); - } - } + setReadColumnIDConf(conf, newConfStr); + } + } - private static void setReadColumnIDConf(Configuration conf, String id) { - if (id == null || id.length() <= 0) { - conf.set(READ_COLUMN_IDS_CONF_STR, ""); - return; - } + private static void setReadColumnIDConf(Configuration conf, String id) { + if (id == null || id.length() <= 0) { + conf.set(READ_COLUMN_IDS_CONF_STR, ""); + return; + } - conf.set(READ_COLUMN_IDS_CONF_STR, id); - } + conf.set(READ_COLUMN_IDS_CONF_STR, id); + } - private static String toReadColumnIDString(ArrayList ids) { - String id = null; - if (ids != null) { - for (int i = 0; i < ids.size(); i++) { - if (i == 0) { - id = "" + ids.get(i); - } else { - id = id + StringUtils.COMMA_STR + ids.get(i); - } - } - } - return id; - } + private static String toReadColumnIDString(ArrayList ids) { + String id = null; + if (ids != null) { + for (int i = 0; i < ids.size(); i++) { + if (i == 0) { + id = "" + ids.get(i); + } else { + id = id + StringUtils.COMMA_STR + ids.get(i); + } + } + } + return id; + } - /** - * Returns an array of column ids(start from zero) which is set in the given - * parameter conf. - */ - public static ArrayList getReadColumnIDs(Configuration conf) { - if( conf == null ) - return new ArrayList(0); - String skips = conf.get(READ_COLUMN_IDS_CONF_STR, ""); - String[] list = StringUtils.split(skips); - ArrayList result = new ArrayList(list.length); - for (int i = 0; i < list.length; i++) { - //it may contain duplicates, remove duplicates - Integer toAdd = Integer.parseInt(list[i]); - if (!result.contains(toAdd)) - result.add(toAdd); - } - return result; - } + /** + * Returns an array of column ids(start from zero) which is set in the given + * parameter conf. + */ + public static ArrayList getReadColumnIDs(Configuration conf) { + if (conf == null) { + return new ArrayList(0); + } + String skips = conf.get(READ_COLUMN_IDS_CONF_STR, ""); + String[] list = StringUtils.split(skips); + ArrayList result = new ArrayList(list.length); + for (String element : list) { + // it may contain duplicates, remove duplicates + Integer toAdd = Integer.parseInt(element); + if (!result.contains(toAdd)) { + result.add(toAdd); + } + } + return result; + } - /** - * Clears the read column ids set in the conf, and will read all columns. - */ - public static void setFullyReadColumns(Configuration conf) { - conf.set(READ_COLUMN_IDS_CONF_STR, ""); - } + /** + * Clears the read column ids set in the conf, and will read all columns. + */ + public static void setFullyReadColumns(Configuration conf) { + conf.set(READ_COLUMN_IDS_CONF_STR, ""); + } } Index: serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/StructTypeInfo.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/StructTypeInfo.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/StructTypeInfo.java (working copy) @@ -25,30 +25,35 @@ import org.apache.hadoop.hive.serde.Constants; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; -/** StructTypeInfo represents the TypeInfo of a struct. - * A struct contains one or more fields each of which has a unique name - * and its own TypeInfo. Different fields can have the same or different - * TypeInfo. - * - * Always use the TypeInfoFactory to create new TypeInfo objects, instead - * of directly creating an instance of this class. +/** + * StructTypeInfo represents the TypeInfo of a struct. A struct contains one or + * more fields each of which has a unique name and its own TypeInfo. Different + * fields can have the same or different TypeInfo. + * + * Always use the TypeInfoFactory to create new TypeInfo objects, instead of + * directly creating an instance of this class. */ -public class StructTypeInfo extends TypeInfo implements Serializable{ +public class StructTypeInfo extends TypeInfo implements Serializable { private static final long serialVersionUID = 1L; - + ArrayList allStructFieldNames; ArrayList allStructFieldTypeInfos; - - /** For java serialization use only. + + /** + * For java serialization use only. */ - public StructTypeInfo() {} + public StructTypeInfo() { + } + @Override public String getTypeName() { StringBuilder sb = new StringBuilder(); sb.append(Constants.STRUCT_TYPE_NAME + "<"); - for(int i=0; i0) sb.append(","); + for (int i = 0; i < allStructFieldNames.size(); i++) { + if (i > 0) { + sb.append(","); + } sb.append(allStructFieldNames.get(i)); sb.append(":"); sb.append(allStructFieldTypeInfos.get(i).getTypeName()); @@ -56,21 +61,24 @@ sb.append(">"); return sb.toString(); } - - /** For java serialization use only. + + /** + * For java serialization use only. */ public void setAllStructFieldNames(ArrayList allStructFieldNames) { this.allStructFieldNames = allStructFieldNames; } - /** For java serialization use only. + /** + * For java serialization use only. */ public void setAllStructFieldTypeInfos( ArrayList allStructFieldTypeInfos) { this.allStructFieldTypeInfos = allStructFieldTypeInfos; } - - /** For TypeInfoFactory use only. + + /** + * For TypeInfoFactory use only. */ StructTypeInfo(List names, List typeInfos) { allStructFieldNames = new ArrayList(); @@ -79,33 +87,36 @@ allStructFieldTypeInfos.addAll(typeInfos); } + @Override public Category getCategory() { return Category.STRUCT; } - + public ArrayList getAllStructFieldNames() { return allStructFieldNames; } - + public ArrayList getAllStructFieldTypeInfos() { return allStructFieldTypeInfos; } - + public TypeInfo getStructFieldTypeInfo(String field) { String fieldLowerCase = field.toLowerCase(); - for(int i=0; i cachedPrimitiveTypeInfo = new HashMap(); + public static TypeInfo getPrimitiveTypeInfo(String typeName) { - if (null == PrimitiveObjectInspectorUtils.getTypeEntryFromTypeName(typeName)) { + if (null == PrimitiveObjectInspectorUtils + .getTypeEntryFromTypeName(typeName)) { throw new RuntimeException("Cannot getPrimitiveTypeInfo for " + typeName); } TypeInfo result = cachedPrimitiveTypeInfo.get(typeName); - if (result == null) { + if (result == null) { result = new PrimitiveTypeInfo(typeName); cachedPrimitiveTypeInfo.put(typeName, result); } @@ -63,36 +60,43 @@ public static final TypeInfo shortTypeInfo = getPrimitiveTypeInfo(Constants.SMALLINT_TYPE_NAME); public static final TypeInfo unknownTypeInfo = getPrimitiveTypeInfo("unknown"); - - public static TypeInfo getPrimitiveTypeInfoFromPrimitiveWritable(Class clazz) { - String typeName = PrimitiveObjectInspectorUtils.getTypeNameFromPrimitiveWritable(clazz); + + public static TypeInfo getPrimitiveTypeInfoFromPrimitiveWritable( + Class clazz) { + String typeName = PrimitiveObjectInspectorUtils + .getTypeNameFromPrimitiveWritable(clazz); if (typeName == null) { - throw new RuntimeException("Internal error: Cannot get typeName for " + clazz); + throw new RuntimeException("Internal error: Cannot get typeName for " + + clazz); } return getPrimitiveTypeInfo(typeName); } public static TypeInfo getPrimitiveTypeInfoFromJavaPrimitive(Class clazz) { - return getPrimitiveTypeInfo(PrimitiveObjectInspectorUtils.getTypeNameFromPrimitiveJava(clazz)); + return getPrimitiveTypeInfo(PrimitiveObjectInspectorUtils + .getTypeNameFromPrimitiveJava(clazz)); } - + static HashMap>, TypeInfo> cachedStructTypeInfo = new HashMap>, TypeInfo>(); - public static TypeInfo getStructTypeInfo(List names, List typeInfos) { + + public static TypeInfo getStructTypeInfo(List names, + List typeInfos) { ArrayList> signature = new ArrayList>(2); signature.add(names); signature.add(typeInfos); TypeInfo result = cachedStructTypeInfo.get(signature); - if (result == null) { + if (result == null) { result = new StructTypeInfo(names, typeInfos); cachedStructTypeInfo.put(signature, result); } return result; } - + static HashMap cachedListTypeInfo = new HashMap(); + public static TypeInfo getListTypeInfo(TypeInfo elementTypeInfo) { TypeInfo result = cachedListTypeInfo.get(elementTypeInfo); - if (result == null) { + if (result == null) { result = new ListTypeInfo(elementTypeInfo); cachedListTypeInfo.put(elementTypeInfo, result); } @@ -100,12 +104,14 @@ } static HashMap, TypeInfo> cachedMapTypeInfo = new HashMap, TypeInfo>(); - public static TypeInfo getMapTypeInfo(TypeInfo keyTypeInfo, TypeInfo valueTypeInfo) { + + public static TypeInfo getMapTypeInfo(TypeInfo keyTypeInfo, + TypeInfo valueTypeInfo) { ArrayList signature = new ArrayList(2); signature.add(keyTypeInfo); signature.add(valueTypeInfo); TypeInfo result = cachedMapTypeInfo.get(signature); - if (result == null) { + if (result == null) { result = new MapTypeInfo(keyTypeInfo, valueTypeInfo); cachedMapTypeInfo.put(signature, result); } Index: serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/PrimitiveTypeInfo.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/PrimitiveTypeInfo.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/PrimitiveTypeInfo.java (working copy) @@ -19,38 +19,41 @@ package org.apache.hadoop.hive.serde2.typeinfo; import java.io.Serializable; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; + import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; - -/** There are limited number of Primitive Types. - * All Primitive Types are defined by TypeInfoFactory.isPrimitiveClass(). - * - * Always use the TypeInfoFactory to create new TypeInfo objects, instead - * of directly creating an instance of this class. +/** + * There are limited number of Primitive Types. All Primitive Types are defined + * by TypeInfoFactory.isPrimitiveClass(). + * + * Always use the TypeInfoFactory to create new TypeInfo objects, instead of + * directly creating an instance of this class. */ public class PrimitiveTypeInfo extends TypeInfo implements Serializable { private static final long serialVersionUID = 1L; - + String typeName; - - /** For java serialization use only. + + /** + * For java serialization use only. */ - public PrimitiveTypeInfo() {} + public PrimitiveTypeInfo() { + } - /** For TypeInfoFactory use only. + /** + * For TypeInfoFactory use only. */ PrimitiveTypeInfo(String typeName) { this.typeName = typeName; } - + /** * Returns the category of this TypeInfo. */ - @Override + @Override public Category getCategory() { return Category.PRIMITIVE; } @@ -58,39 +61,40 @@ public PrimitiveCategory getPrimitiveCategory() { return PrimitiveObjectInspectorUtils.getTypeEntryFromTypeName(typeName).primitiveCategory; } - + public Class getPrimitiveWritableClass() { return PrimitiveObjectInspectorUtils.getTypeEntryFromTypeName(typeName).primitiveWritableClass; } - + public Class getPrimitiveJavaClass() { return PrimitiveObjectInspectorUtils.getTypeEntryFromTypeName(typeName).primitiveJavaClass; } - - + // The following 2 methods are for java serialization use only. public void setTypeName(String typeName) { this.typeName = typeName; } + @Override public String getTypeName() { return typeName; } /** - * Compare if 2 TypeInfos are the same. - * We use TypeInfoFactory to cache TypeInfos, so we only - * need to compare the Object pointer. + * Compare if 2 TypeInfos are the same. We use TypeInfoFactory to cache + * TypeInfos, so we only need to compare the Object pointer. */ + @Override public boolean equals(Object other) { return this == other; } - + /** * Generate the hashCode for this TypeInfo. */ + @Override public int hashCode() { return typeName.hashCode(); } - + } Index: serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/ListTypeInfo.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/ListTypeInfo.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/ListTypeInfo.java (working copy) @@ -19,50 +19,61 @@ package org.apache.hadoop.hive.serde2.typeinfo; import java.io.Serializable; + import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; -/** A List Type has homogeneous elements. All elements of the List has - * the same TypeInfo which is returned by getListElementTypeInfo. - * - * Always use the TypeInfoFactory to create new TypeInfo objects, instead - * of directly creating an instance of this class. +/** + * A List Type has homogeneous elements. All elements of the List has the same + * TypeInfo which is returned by getListElementTypeInfo. + * + * Always use the TypeInfoFactory to create new TypeInfo objects, instead of + * directly creating an instance of this class. */ public class ListTypeInfo extends TypeInfo implements Serializable { private static final long serialVersionUID = 1L; TypeInfo listElementTypeInfo; - - /** For java serialization use only. + + /** + * For java serialization use only. */ - public ListTypeInfo() {} - + public ListTypeInfo() { + } + + @Override public String getTypeName() { - return org.apache.hadoop.hive.serde.Constants.LIST_TYPE_NAME - + "<" + listElementTypeInfo.getTypeName() + ">"; + return org.apache.hadoop.hive.serde.Constants.LIST_TYPE_NAME + "<" + + listElementTypeInfo.getTypeName() + ">"; } - - /** For java serialization use only. + + /** + * For java serialization use only. */ public void setListElementTypeInfo(TypeInfo listElementTypeInfo) { this.listElementTypeInfo = listElementTypeInfo; } - - /** For TypeInfoFactory use only. + + /** + * For TypeInfoFactory use only. */ ListTypeInfo(TypeInfo elementTypeInfo) { - this.listElementTypeInfo = elementTypeInfo; + listElementTypeInfo = elementTypeInfo; } + @Override public Category getCategory() { return Category.LIST; - } + } public TypeInfo getListElementTypeInfo() { return listElementTypeInfo; } + @Override public boolean equals(Object other) { - if (this == other) return true; + if (this == other) { + return true; + } if (!(other instanceof ListTypeInfo)) { return false; } @@ -70,9 +81,10 @@ return o.getCategory().equals(getCategory()) && o.getListElementTypeInfo().equals(getListElementTypeInfo()); } - + + @Override public int hashCode() { return listElementTypeInfo.hashCode(); } - + } Index: serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/MapTypeInfo.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/MapTypeInfo.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/MapTypeInfo.java (working copy) @@ -22,52 +22,59 @@ import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; -/** A Map Type has homogeneous keys and homogeneous values. - * All keys of the Map have the same TypeInfo, which is returned by - * getMapKeyTypeInfo(); and all values of the Map has the same TypeInfo, - * which is returned by getMapValueTypeInfo(). - * - * Always use the TypeInfoFactory to create new TypeInfo objects, instead - * of directly creating an instance of this class. +/** + * A Map Type has homogeneous keys and homogeneous values. All keys of the Map + * have the same TypeInfo, which is returned by getMapKeyTypeInfo(); and all + * values of the Map has the same TypeInfo, which is returned by + * getMapValueTypeInfo(). + * + * Always use the TypeInfoFactory to create new TypeInfo objects, instead of + * directly creating an instance of this class. */ -public class MapTypeInfo extends TypeInfo implements Serializable{ +public class MapTypeInfo extends TypeInfo implements Serializable { private static final long serialVersionUID = 1L; - + TypeInfo mapKeyTypeInfo; TypeInfo mapValueTypeInfo; - - /** For java serialization use only. + + /** + * For java serialization use only. */ - public MapTypeInfo() {} - + public MapTypeInfo() { + } + + @Override public String getTypeName() { - return org.apache.hadoop.hive.serde.Constants.MAP_TYPE_NAME - + "<" + mapKeyTypeInfo.getTypeName() + "," - + mapValueTypeInfo.getTypeName() + ">"; + return org.apache.hadoop.hive.serde.Constants.MAP_TYPE_NAME + "<" + + mapKeyTypeInfo.getTypeName() + "," + mapValueTypeInfo.getTypeName() + + ">"; } - - /** For java serialization use only. + + /** + * For java serialization use only. */ public void setMapKeyTypeInfo(TypeInfo mapKeyTypeInfo) { this.mapKeyTypeInfo = mapKeyTypeInfo; } - /** For java serialization use only. + /** + * For java serialization use only. */ public void setMapValueTypeInfo(TypeInfo mapValueTypeInfo) { this.mapValueTypeInfo = mapValueTypeInfo; } - + // For TypeInfoFactory use only MapTypeInfo(TypeInfo keyTypeInfo, TypeInfo valueTypeInfo) { - this.mapKeyTypeInfo = keyTypeInfo; - this.mapValueTypeInfo = valueTypeInfo; + mapKeyTypeInfo = keyTypeInfo; + mapValueTypeInfo = valueTypeInfo; } + @Override public Category getCategory() { return Category.MAP; - } + } public TypeInfo getMapKeyTypeInfo() { return mapKeyTypeInfo; @@ -77,8 +84,11 @@ return mapValueTypeInfo; } + @Override public boolean equals(Object other) { - if (this == other) return true; + if (this == other) { + return true; + } if (!(other instanceof MapTypeInfo)) { return false; } @@ -87,10 +97,10 @@ && o.getMapKeyTypeInfo().equals(getMapKeyTypeInfo()) && o.getMapValueTypeInfo().equals(getMapValueTypeInfo()); } - + + @Override public int hashCode() { return mapKeyTypeInfo.hashCode() ^ mapValueTypeInfo.hashCode(); } - - + } Index: serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfo.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfo.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfo.java (working copy) @@ -19,46 +19,51 @@ package org.apache.hadoop.hive.serde2.typeinfo; import java.io.Serializable; + import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; /** - * Stores information about a type. - * Always use the TypeInfoFactory to create new TypeInfo objects. + * Stores information about a type. Always use the TypeInfoFactory to create new + * TypeInfo objects. * - * We support 4 categories of types: - * 1. Primitive objects (String, Number, etc) - * 2. List objects (a list of objects of a single type) - * 3. Map objects (a map from objects of one type to objects of another type) - * 4. Struct objects (a list of fields with names and their own types) + * We support 4 categories of types: 1. Primitive objects (String, Number, etc) + * 2. List objects (a list of objects of a single type) 3. Map objects (a map + * from objects of one type to objects of another type) 4. Struct objects (a + * list of fields with names and their own types) */ public abstract class TypeInfo implements Serializable { - protected TypeInfo() {} + protected TypeInfo() { + } /** - * The Category of this TypeInfo. - * Possible values are Primitive, List, Map and Struct, which corresponds - * to the 4 sub-classes of TypeInfo. + * The Category of this TypeInfo. Possible values are Primitive, List, Map and + * Struct, which corresponds to the 4 sub-classes of TypeInfo. */ public abstract Category getCategory(); - + /** * A String representation of the TypeInfo. */ public abstract String getTypeName(); - + + @Override public String toString() { return getTypeName(); } + @Override public boolean equals(Object o) { - if (!(o instanceof TypeInfo)) - return false; - TypeInfo dest = (TypeInfo)o; - if (getCategory() != dest.getCategory()) - return false; - if (getTypeName() != dest.getTypeName()) - return false; - return true; + if (!(o instanceof TypeInfo)) { + return false; + } + TypeInfo dest = (TypeInfo) o; + if (getCategory() != dest.getCategory()) { + return false; + } + if (getTypeName() != dest.getTypeName()) { + return false; + } + return true; } } Index: serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfoUtils.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfoUtils.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfoUtils.java (working copy) @@ -25,111 +25,121 @@ import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils.PrimitiveTypeEntry; public class TypeInfoUtils { - + /** - * Return the extended TypeInfo from a Java type. - * By extended TypeInfo, we allow unknownType for java.lang.Object. - * @param t The Java type. - * @param m The method, only used for generating error messages. + * Return the extended TypeInfo from a Java type. By extended TypeInfo, we + * allow unknownType for java.lang.Object. + * + * @param t + * The Java type. + * @param m + * The method, only used for generating error messages. */ private static TypeInfo getExtendedTypeInfoFromJavaType(Type t, Method m) { if (t == Object.class) { - return TypeInfoFactory.unknownTypeInfo; + return TypeInfoFactory.unknownTypeInfo; } - + if (t instanceof ParameterizedType) { - ParameterizedType pt = (ParameterizedType)t; + ParameterizedType pt = (ParameterizedType) t; // List? - if (List.class == (Class)pt.getRawType() - || ArrayList.class == (Class)pt.getRawType()) { - return TypeInfoFactory.getListTypeInfo( - getExtendedTypeInfoFromJavaType(pt.getActualTypeArguments()[0], m)); + if (List.class == (Class) pt.getRawType() + || ArrayList.class == (Class) pt.getRawType()) { + return TypeInfoFactory.getListTypeInfo(getExtendedTypeInfoFromJavaType( + pt.getActualTypeArguments()[0], m)); } // Map? - if (Map.class == (Class)pt.getRawType() - || HashMap.class == (Class)pt.getRawType()) { - return TypeInfoFactory.getMapTypeInfo( - getExtendedTypeInfoFromJavaType(pt.getActualTypeArguments()[0], m), + if (Map.class == (Class) pt.getRawType() + || HashMap.class == (Class) pt.getRawType()) { + return TypeInfoFactory.getMapTypeInfo(getExtendedTypeInfoFromJavaType( + pt.getActualTypeArguments()[0], m), getExtendedTypeInfoFromJavaType(pt.getActualTypeArguments()[1], m)); } - // Otherwise convert t to RawType so we will fall into the following if block. + // Otherwise convert t to RawType so we will fall into the following if + // block. t = pt.getRawType(); } - + // Must be a class. if (!(t instanceof Class)) { - throw new RuntimeException("Hive does not understand type " + t + " from " + m); + throw new RuntimeException("Hive does not understand type " + t + + " from " + m); } - Class c = (Class)t; - + Class c = (Class) t; + // Java Primitive Type? if (PrimitiveObjectInspectorUtils.isPrimitiveJavaType(c)) { - return TypeInfoUtils.getTypeInfoFromObjectInspector( - PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector( - PrimitiveObjectInspectorUtils.getTypeEntryFromPrimitiveJavaType(c).primitiveCategory)); + return TypeInfoUtils + .getTypeInfoFromObjectInspector(PrimitiveObjectInspectorFactory + .getPrimitiveJavaObjectInspector(PrimitiveObjectInspectorUtils + .getTypeEntryFromPrimitiveJavaType(c).primitiveCategory)); } // Java Primitive Class? if (PrimitiveObjectInspectorUtils.isPrimitiveJavaClass(c)) { - return TypeInfoUtils.getTypeInfoFromObjectInspector( - PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector( - PrimitiveObjectInspectorUtils.getTypeEntryFromPrimitiveJavaClass(c).primitiveCategory)); + return TypeInfoUtils + .getTypeInfoFromObjectInspector(PrimitiveObjectInspectorFactory + .getPrimitiveJavaObjectInspector(PrimitiveObjectInspectorUtils + .getTypeEntryFromPrimitiveJavaClass(c).primitiveCategory)); } - + // Primitive Writable class? if (PrimitiveObjectInspectorUtils.isPrimitiveWritableClass(c)) { - return TypeInfoUtils.getTypeInfoFromObjectInspector( - PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector( - PrimitiveObjectInspectorUtils.getTypeEntryFromPrimitiveWritableClass(c).primitiveCategory)); + return TypeInfoUtils + .getTypeInfoFromObjectInspector(PrimitiveObjectInspectorFactory + .getPrimitiveWritableObjectInspector(PrimitiveObjectInspectorUtils + .getTypeEntryFromPrimitiveWritableClass(c).primitiveCategory)); } - + // Must be a struct Field[] fields = ObjectInspectorUtils.getDeclaredNonStaticFields(c); ArrayList fieldNames = new ArrayList(fields.length); ArrayList fieldTypeInfos = new ArrayList(fields.length); - for(int i=0; i[]). Otherwise return null. + * Returns the array element type, if the Type is an array (Object[]), or + * GenericArrayType (Map[]). Otherwise return null. */ public static Type getArrayElementType(Type t) { - if (t instanceof Class - && ((Class)t).isArray()) { - Class arrayClass = (Class)t; + if (t instanceof Class && ((Class) t).isArray()) { + Class arrayClass = (Class) t; return arrayClass.getComponentType(); } else if (t instanceof GenericArrayType) { - GenericArrayType arrayType = (GenericArrayType)t; + GenericArrayType arrayType = (GenericArrayType) t; return arrayType.getGenericComponentType(); } return null; } - + /** * Get the parameter TypeInfo for a method. - * @param size In case the last parameter of Method is an array, we will try to return a - * List with the specified size by repeating the element of the array - * at the end. - * In case the size is smaller than the minimum possible number of arguments - * for the method, null will be returned. + * + * @param size + * In case the last parameter of Method is an array, we will try to + * return a List with the specified size by repeating the + * element of the array at the end. In case the size is smaller than + * the minimum possible number of arguments for the method, null will + * be returned. */ public static List getParameterTypeInfos(Method m, int size) { Type[] methodParameterTypes = m.getGenericParameterTypes(); - + // Whether the method takes variable-length arguments - // Whether the method takes an array like Object[], + // Whether the method takes an array like Object[], // or String[] etc in the last argument. - Type lastParaElementType = TypeInfoUtils.getArrayElementType( - methodParameterTypes.length == 0 ? null : - methodParameterTypes[methodParameterTypes.length-1]); + Type lastParaElementType = TypeInfoUtils + .getArrayElementType(methodParameterTypes.length == 0 ? null + : methodParameterTypes[methodParameterTypes.length - 1]); boolean isVariableLengthArgument = (lastParaElementType != null); - + List typeInfos = null; if (!isVariableLengthArgument) { // Normal case, no variable-length arguments @@ -137,8 +147,8 @@ return null; } typeInfos = new ArrayList(methodParameterTypes.length); - for (int i = 0; i < methodParameterTypes.length; i++) { - typeInfos.add(getExtendedTypeInfoFromJavaType(methodParameterTypes[i], m)); + for (Type methodParameterType : methodParameterTypes) { + typeInfos.add(getExtendedTypeInfoFromJavaType(methodParameterType, m)); } } else { // Variable-length arguments @@ -147,7 +157,8 @@ } typeInfos = new ArrayList(size); for (int i = 0; i < methodParameterTypes.length - 1; i++) { - typeInfos.add(getExtendedTypeInfoFromJavaType(methodParameterTypes[i], m)); + typeInfos.add(getExtendedTypeInfoFromJavaType(methodParameterTypes[i], + m)); } for (int i = methodParameterTypes.length - 1; i < size; i++) { typeInfos.add(getExtendedTypeInfoFromJavaType(lastParaElementType, m)); @@ -155,46 +166,49 @@ } return typeInfos; } + /** - * Parse a recursive TypeInfo list String. - * For example, the following inputs are valid inputs: - * "int,string,map,list>>,list>" + * Parse a recursive TypeInfo list String. For example, the following inputs + * are valid inputs:"int,string,map,list>>,list>" * The separators between TypeInfos can be ",", ":", or ";". * - * In order to use this class: - * TypeInfoParser parser = new TypeInfoParser("int,string"); - * ArrayList typeInfos = parser.parseTypeInfos(); + * In order to use this class: TypeInfoParser parser = new + * TypeInfoParser("int,string"); ArrayList typeInfos = + * parser.parseTypeInfos(); */ private static class TypeInfoParser { - + private static class Token { public int position; public String text; public boolean isType; + + @Override public String toString() { return "" + position + ":" + text; } }; - + private static boolean isTypeChar(char c) { return Character.isLetterOrDigit(c) || c == '_' || c == '.'; } - + /** - * Tokenize the typeInfoString. - * The rule is simple: all consecutive alphadigits and '_', '.' are in one - * token, and all other characters are one character per token. + * Tokenize the typeInfoString. The rule is simple: all consecutive + * alphadigits and '_', '.' are in one token, and all other characters are + * one character per token. * - * tokenize("map") should return ["map","<","int",",","string",">"] + * tokenize("map") should return + * ["map","<","int",",","string",">"] */ private static ArrayList tokenize(String typeInfoString) { ArrayList tokens = new ArrayList(0); int begin = 0; int end = 1; while (end <= typeInfoString.length()) { - // last character ends a token? - if (end == typeInfoString.length() - || !isTypeChar(typeInfoString.charAt(end-1)) + // last character ends a token? + if (end == typeInfoString.length() + || !isTypeChar(typeInfoString.charAt(end - 1)) || !isTypeChar(typeInfoString.charAt(end))) { Token t = new Token(); t.position = begin; @@ -202,22 +216,22 @@ t.isType = isTypeChar(typeInfoString.charAt(begin)); tokens.add(t); begin = end; - } - end ++; + } + end++; } return tokens; } - + public TypeInfoParser(String typeInfoString) { this.typeInfoString = typeInfoString; - this.typeInfoTokens = tokenize(typeInfoString); + typeInfoTokens = tokenize(typeInfoString); } - - private String typeInfoString; - private ArrayList typeInfoTokens; + + private final String typeInfoString; + private final ArrayList typeInfoTokens; private ArrayList typeInfos; private int iToken; - + public ArrayList parseTypeInfos() throws IllegalArgumentException { typeInfos = new ArrayList(); iToken = 0; @@ -225,61 +239,70 @@ typeInfos.add(parseType()); if (iToken < typeInfoTokens.size()) { Token separator = typeInfoTokens.get(iToken); - if (",".equals(separator.text) || ";".equals(separator.text) || ":".equals(separator.text)) { - iToken ++; + if (",".equals(separator.text) || ";".equals(separator.text) + || ":".equals(separator.text)) { + iToken++; } else { - throw new IllegalArgumentException("Error: ',', ':', or ';' expected at position " - + separator.position + " from '" + typeInfoString + "' " + typeInfoTokens ); + throw new IllegalArgumentException( + "Error: ',', ':', or ';' expected at position " + + separator.position + " from '" + typeInfoString + "' " + + typeInfoTokens); } } } return typeInfos; } - + private Token expect(String item) { return expect(item, null); } - + private Token expect(String item, String alternative) { if (iToken >= typeInfoTokens.size()) { - throw new IllegalArgumentException("Error: " + item + " expected at the end of '" - + typeInfoString + "'" ); + throw new IllegalArgumentException("Error: " + item + + " expected at the end of '" + typeInfoString + "'"); } Token t = typeInfoTokens.get(iToken); if (item.equals("type")) { if (!Constants.LIST_TYPE_NAME.equals(t.text) && !Constants.MAP_TYPE_NAME.equals(t.text) && !Constants.STRUCT_TYPE_NAME.equals(t.text) - && null == PrimitiveObjectInspectorUtils.getTypeEntryFromTypeName(t.text) + && null == PrimitiveObjectInspectorUtils + .getTypeEntryFromTypeName(t.text) && !t.text.equals(alternative)) { - throw new IllegalArgumentException("Error: " + item + " expected at the position " - + t.position + " of '" + typeInfoString + "' but '" + t.text + "' is found." ); + throw new IllegalArgumentException("Error: " + item + + " expected at the position " + t.position + " of '" + + typeInfoString + "' but '" + t.text + "' is found."); } } else if (item.equals("name")) { if (!t.isType && !t.text.equals(alternative)) { - throw new IllegalArgumentException("Error: " + item + " expected at the position " - + t.position + " of '" + typeInfoString + "' but '" + t.text + "' is found." ); + throw new IllegalArgumentException("Error: " + item + + " expected at the position " + t.position + " of '" + + typeInfoString + "' but '" + t.text + "' is found."); } } else { if (!item.equals(t.text) && !t.text.equals(alternative)) { - throw new IllegalArgumentException("Error: " + item + " expected at the position " - + t.position + " of '" + typeInfoString + "' but '" + t.text + "' is found." ); + throw new IllegalArgumentException("Error: " + item + + " expected at the position " + t.position + " of '" + + typeInfoString + "' but '" + t.text + "' is found."); } } - iToken ++; + iToken++; return t; } - + private TypeInfo parseType() { - + Token t = expect("type"); - + // Is this a primitive type? - PrimitiveTypeEntry primitiveType = PrimitiveObjectInspectorUtils.getTypeEntryFromTypeName(t.text); - if (primitiveType != null && !primitiveType.primitiveCategory.equals(PrimitiveCategory.UNKNOWN)) { + PrimitiveTypeEntry primitiveType = PrimitiveObjectInspectorUtils + .getTypeEntryFromTypeName(t.text); + if (primitiveType != null + && !primitiveType.primitiveCategory.equals(PrimitiveCategory.UNKNOWN)) { return TypeInfoFactory.getPrimitiveTypeInfo(primitiveType.typeName); } - + // Is this a list type? if (Constants.LIST_TYPE_NAME.equals(t.text)) { expect("<"); @@ -287,7 +310,7 @@ expect(">"); return TypeInfoFactory.getListTypeInfo(listElementType); } - + // Is this a map type? if (Constants.MAP_TYPE_NAME.equals(t.text)) { expect("<"); @@ -297,7 +320,7 @@ expect(">"); return TypeInfoFactory.getMapTypeInfo(mapKeyType, mapValueType); } - + // Is this a struct type? if (Constants.STRUCT_TYPE_NAME.equals(t.text)) { ArrayList fieldNames = new ArrayList(); @@ -319,164 +342,190 @@ expect(":"); fieldTypeInfos.add(parseType()); } while (true); - + return TypeInfoFactory.getStructTypeInfo(fieldNames, fieldTypeInfos); } - - throw new RuntimeException("Internal error parsing position " + t.position + " of '" - + typeInfoString + "'"); + + throw new RuntimeException("Internal error parsing position " + + t.position + " of '" + typeInfoString + "'"); } - + } static HashMap cachedStandardObjectInspector = new HashMap(); + /** - * Returns the standard object inspector that can be used to translate an object of that typeInfo - * to a standard object type. + * Returns the standard object inspector that can be used to translate an + * object of that typeInfo to a standard object type. */ - public static ObjectInspector getStandardWritableObjectInspectorFromTypeInfo(TypeInfo typeInfo) { + public static ObjectInspector getStandardWritableObjectInspectorFromTypeInfo( + TypeInfo typeInfo) { ObjectInspector result = cachedStandardObjectInspector.get(typeInfo); if (result == null) { - switch(typeInfo.getCategory()) { - case PRIMITIVE: { - result = PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector( - ((PrimitiveTypeInfo)typeInfo).getPrimitiveCategory()); - break; + switch (typeInfo.getCategory()) { + case PRIMITIVE: { + result = PrimitiveObjectInspectorFactory + .getPrimitiveWritableObjectInspector(((PrimitiveTypeInfo) typeInfo) + .getPrimitiveCategory()); + break; + } + case LIST: { + ObjectInspector elementObjectInspector = getStandardWritableObjectInspectorFromTypeInfo(((ListTypeInfo) typeInfo) + .getListElementTypeInfo()); + result = ObjectInspectorFactory + .getStandardListObjectInspector(elementObjectInspector); + break; + } + case MAP: { + MapTypeInfo mapTypeInfo = (MapTypeInfo) typeInfo; + ObjectInspector keyObjectInspector = getStandardWritableObjectInspectorFromTypeInfo(mapTypeInfo + .getMapKeyTypeInfo()); + ObjectInspector valueObjectInspector = getStandardWritableObjectInspectorFromTypeInfo(mapTypeInfo + .getMapValueTypeInfo()); + result = ObjectInspectorFactory.getStandardMapObjectInspector( + keyObjectInspector, valueObjectInspector); + break; + } + case STRUCT: { + StructTypeInfo structTypeInfo = (StructTypeInfo) typeInfo; + List fieldNames = structTypeInfo.getAllStructFieldNames(); + List fieldTypeInfos = structTypeInfo + .getAllStructFieldTypeInfos(); + List fieldObjectInspectors = new ArrayList( + fieldTypeInfos.size()); + for (int i = 0; i < fieldTypeInfos.size(); i++) { + fieldObjectInspectors + .add(getStandardWritableObjectInspectorFromTypeInfo(fieldTypeInfos + .get(i))); } - case LIST: { - ObjectInspector elementObjectInspector = getStandardWritableObjectInspectorFromTypeInfo( - ((ListTypeInfo)typeInfo).getListElementTypeInfo()); - result = ObjectInspectorFactory.getStandardListObjectInspector(elementObjectInspector); - break; - } - case MAP: { - MapTypeInfo mapTypeInfo = (MapTypeInfo)typeInfo; - ObjectInspector keyObjectInspector = getStandardWritableObjectInspectorFromTypeInfo(mapTypeInfo.getMapKeyTypeInfo()); - ObjectInspector valueObjectInspector = getStandardWritableObjectInspectorFromTypeInfo(mapTypeInfo.getMapValueTypeInfo()); - result = ObjectInspectorFactory.getStandardMapObjectInspector(keyObjectInspector, valueObjectInspector); - break; - } - case STRUCT: { - StructTypeInfo structTypeInfo = (StructTypeInfo)typeInfo; - List fieldNames = structTypeInfo.getAllStructFieldNames(); - List fieldTypeInfos = structTypeInfo.getAllStructFieldTypeInfos(); - List fieldObjectInspectors = new ArrayList(fieldTypeInfos.size()); - for(int i=0; i cachedStandardJavaObjectInspector = new HashMap(); - - static HashMap cachedStandardJavaObjectInspector = new HashMap(); /** - * Returns the standard object inspector that can be used to translate an object of that typeInfo - * to a standard object type. + * Returns the standard object inspector that can be used to translate an + * object of that typeInfo to a standard object type. */ - public static ObjectInspector getStandardJavaObjectInspectorFromTypeInfo(TypeInfo typeInfo) { + public static ObjectInspector getStandardJavaObjectInspectorFromTypeInfo( + TypeInfo typeInfo) { ObjectInspector result = cachedStandardJavaObjectInspector.get(typeInfo); if (result == null) { - switch(typeInfo.getCategory()) { - case PRIMITIVE: { - // NOTE: we use JavaPrimitiveObjectInspector instead of StandardPrimitiveObjectInspector - result = PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector( - PrimitiveObjectInspectorUtils.getTypeEntryFromTypeName(typeInfo.getTypeName()).primitiveCategory); - break; - } - case LIST: { - ObjectInspector elementObjectInspector = getStandardJavaObjectInspectorFromTypeInfo( - ((ListTypeInfo)typeInfo).getListElementTypeInfo()); - result = ObjectInspectorFactory.getStandardListObjectInspector(elementObjectInspector); - break; - } - case MAP: { - MapTypeInfo mapTypeInfo = (MapTypeInfo)typeInfo; - ObjectInspector keyObjectInspector = getStandardJavaObjectInspectorFromTypeInfo(mapTypeInfo.getMapKeyTypeInfo()); - ObjectInspector valueObjectInspector = getStandardJavaObjectInspectorFromTypeInfo(mapTypeInfo.getMapValueTypeInfo()); - result = ObjectInspectorFactory.getStandardMapObjectInspector(keyObjectInspector, valueObjectInspector); - break; - } - case STRUCT: { - StructTypeInfo strucTypeInfo = (StructTypeInfo)typeInfo; - List fieldNames = strucTypeInfo.getAllStructFieldNames(); - List fieldTypeInfos = strucTypeInfo.getAllStructFieldTypeInfos(); - List fieldObjectInspectors = new ArrayList(fieldTypeInfos.size()); - for(int i=0; i fields = soi.getAllStructFieldRefs(); - List fieldNames = new ArrayList(fields.size()); - List fieldTypeInfos = new ArrayList(fields.size()); - for(StructField f : fields) { - fieldNames.add(f.getFieldName()); - fieldTypeInfos.add(getTypeInfoFromObjectInspector(f.getFieldObjectInspector())); + StructTypeInfo strucTypeInfo = (StructTypeInfo) typeInfo; + List fieldNames = strucTypeInfo.getAllStructFieldNames(); + List fieldTypeInfos = strucTypeInfo + .getAllStructFieldTypeInfos(); + List fieldObjectInspectors = new ArrayList( + fieldTypeInfos.size()); + for (int i = 0; i < fieldTypeInfos.size(); i++) { + fieldObjectInspectors + .add(getStandardJavaObjectInspectorFromTypeInfo(fieldTypeInfos + .get(i))); } - result = TypeInfoFactory.getStructTypeInfo(fieldNames, fieldTypeInfos); + result = ObjectInspectorFactory.getStandardStructObjectInspector( + fieldNames, fieldObjectInspectors); break; } default: { - throw new RuntimeException("Unknown ObjectInspector category!"); + result = null; } + } + cachedStandardJavaObjectInspector.put(typeInfo, result); } return result; } - + + /** + * Get the TypeInfo object from the ObjectInspector object by recursively + * going into the ObjectInspector structure. + */ + public static TypeInfo getTypeInfoFromObjectInspector(ObjectInspector oi) { + // OPTIMIZATION for later. + // if (oi instanceof TypeInfoBasedObjectInspector) { + // TypeInfoBasedObjectInspector typeInfoBasedObjectInspector = + // (ObjectInspector)oi; + // return typeInfoBasedObjectInspector.getTypeInfo(); + // } + + // Recursively going into ObjectInspector structure + TypeInfo result = null; + switch (oi.getCategory()) { + case PRIMITIVE: { + PrimitiveObjectInspector poi = (PrimitiveObjectInspector) oi; + result = TypeInfoFactory.getPrimitiveTypeInfo(poi.getTypeName()); + break; + } + case LIST: { + ListObjectInspector loi = (ListObjectInspector) oi; + result = TypeInfoFactory + .getListTypeInfo(getTypeInfoFromObjectInspector(loi + .getListElementObjectInspector())); + break; + } + case MAP: { + MapObjectInspector moi = (MapObjectInspector) oi; + result = TypeInfoFactory.getMapTypeInfo( + getTypeInfoFromObjectInspector(moi.getMapKeyObjectInspector()), + getTypeInfoFromObjectInspector(moi.getMapValueObjectInspector())); + break; + } + case STRUCT: { + StructObjectInspector soi = (StructObjectInspector) oi; + List fields = soi.getAllStructFieldRefs(); + List fieldNames = new ArrayList(fields.size()); + List fieldTypeInfos = new ArrayList(fields.size()); + for (StructField f : fields) { + fieldNames.add(f.getFieldName()); + fieldTypeInfos.add(getTypeInfoFromObjectInspector(f + .getFieldObjectInspector())); + } + result = TypeInfoFactory.getStructTypeInfo(fieldNames, fieldTypeInfos); + break; + } + default: { + throw new RuntimeException("Unknown ObjectInspector category!"); + } + } + return result; + } + public static ArrayList getTypeInfosFromTypeString(String typeString) { TypeInfoParser parser = new TypeInfoParser(typeString); return parser.parseTypeInfos(); Index: serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/MetadataListStructObjectInspector.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/MetadataListStructObjectInspector.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/MetadataListStructObjectInspector.java (working copy) @@ -27,21 +27,24 @@ import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; /** - * StructObjectInspector works on struct data that is stored as a Java List or Java Array object. - * Basically, the fields are stored sequentially in the List object. + * StructObjectInspector works on struct data that is stored as a Java List or + * Java Array object. Basically, the fields are stored sequentially in the List + * object. * - * The names of the struct fields and the internal structure of the struct fields are specified in - * the ctor of the StructObjectInspector. + * The names of the struct fields and the internal structure of the struct + * fields are specified in the ctor of the StructObjectInspector. * */ -public class MetadataListStructObjectInspector extends StandardStructObjectInspector { +public class MetadataListStructObjectInspector extends + StandardStructObjectInspector { - static HashMap, MetadataListStructObjectInspector> cached - = new HashMap, MetadataListStructObjectInspector>(); -// public static MetadataListStructObjectInspector getInstance(int fields) { -// return getInstance(ObjectInspectorUtils.getIntegerArray(fields)); -// } - public static MetadataListStructObjectInspector getInstance(List columnNames) { + static HashMap, MetadataListStructObjectInspector> cached = new HashMap, MetadataListStructObjectInspector>(); + + // public static MetadataListStructObjectInspector getInstance(int fields) { + // return getInstance(ObjectInspectorUtils.getIntegerArray(fields)); + // } + public static MetadataListStructObjectInspector getInstance( + List columnNames) { MetadataListStructObjectInspector result = cached.get(columnNames); if (result == null) { result = new MetadataListStructObjectInspector(columnNames); @@ -52,27 +55,31 @@ static ArrayList getFieldObjectInspectors(int fields) { ArrayList r = new ArrayList(fields); - for(int i=0; i columnNames) { super(columnNames, getFieldObjectInspectors(columnNames.size())); } - + // Get col object out + @Override public Object getStructFieldData(Object data, StructField fieldRef) { if (data instanceof ColumnSet) { - data = ((ColumnSet)data).col; + data = ((ColumnSet) data).col; } return super.getStructFieldData(data, fieldRef); } + // Get col object out + @Override public List getStructFieldsDataAsList(Object data) { if (data instanceof ColumnSet) { - data = ((ColumnSet)data).col; + data = ((ColumnSet) data).col; } return super.getStructFieldsDataAsList(data); } Index: serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StructObjectInspector.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StructObjectInspector.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StructObjectInspector.java (working copy) @@ -19,34 +19,40 @@ import java.util.List; - public abstract class StructObjectInspector implements ObjectInspector { // ** Methods that does not need a data object ** - /** Returns all the fields. + /** + * Returns all the fields. */ abstract public List getAllStructFieldRefs(); - /** Look up a field. + /** + * Look up a field. */ abstract public StructField getStructFieldRef(String fieldName); // ** Methods that need a data object ** - /** returns null for data = null. + /** + * returns null for data = null. */ abstract public Object getStructFieldData(Object data, StructField fieldRef); - /** returns null for data = null. + /** + * returns null for data = null. */ abstract public List getStructFieldsDataAsList(Object data); - + + @Override public String toString() { StringBuilder sb = new StringBuilder(); List fields = getAllStructFieldRefs(); sb.append(getClass().getName()); sb.append("<"); - for (int i=0; i0) sb.append(","); + for (int i = 0; i < fields.size(); i++) { + if (i > 0) { + sb.append(","); + } sb.append(fields.get(i).getFieldObjectInspector().toString()); } sb.append(">"); Index: serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ColumnarStructObjectInspector.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ColumnarStructObjectInspector.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ColumnarStructObjectInspector.java (working copy) @@ -65,6 +65,7 @@ return fieldObjectInspector; } + @Override public String toString() { return "" + fieldID + ":" + fieldName; } Index: serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorConverters.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorConverters.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorConverters.java (working copy) @@ -33,7 +33,6 @@ import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableShortObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableStringObjectInspector; - public class ObjectInspectorConverters { /** @@ -42,84 +41,88 @@ public static interface Converter { public Object convert(Object input); } - + public static class IdentityConverter implements Converter { public Object convert(Object input) { return input; } } - + /** - * Returns a converter that converts objects from one OI to another OI. - * The returned (converted) object belongs to this converter, so that it can be reused - * across different calls. + * Returns a converter that converts objects from one OI to another OI. The + * returned (converted) object belongs to this converter, so that it can be + * reused across different calls. */ - public static Converter getConverter(ObjectInspector inputOI, ObjectInspector outputOI) { - // If the inputOI is the same as the outputOI, just return an IdentityConverter. + public static Converter getConverter(ObjectInspector inputOI, + ObjectInspector outputOI) { + // If the inputOI is the same as the outputOI, just return an + // IdentityConverter. if (inputOI == outputOI) { return new IdentityConverter(); } switch (outputOI.getCategory()) { - case PRIMITIVE: - switch (((PrimitiveObjectInspector)outputOI).getPrimitiveCategory()) { - case BOOLEAN: - return new PrimitiveObjectInspectorConverter.BooleanConverter( - (PrimitiveObjectInspector)inputOI, - (SettableBooleanObjectInspector)outputOI); - case BYTE: - return new PrimitiveObjectInspectorConverter.ByteConverter( - (PrimitiveObjectInspector)inputOI, - (SettableByteObjectInspector)outputOI); - case SHORT: - return new PrimitiveObjectInspectorConverter.ShortConverter( - (PrimitiveObjectInspector)inputOI, - (SettableShortObjectInspector)outputOI); - case INT: - return new PrimitiveObjectInspectorConverter.IntConverter( - (PrimitiveObjectInspector)inputOI, - (SettableIntObjectInspector)outputOI); - case LONG: - return new PrimitiveObjectInspectorConverter.LongConverter( - (PrimitiveObjectInspector)inputOI, - (SettableLongObjectInspector)outputOI); - case FLOAT: - return new PrimitiveObjectInspectorConverter.FloatConverter( - (PrimitiveObjectInspector)inputOI, - (SettableFloatObjectInspector)outputOI); - case DOUBLE: - return new PrimitiveObjectInspectorConverter.DoubleConverter( - (PrimitiveObjectInspector)inputOI, - (SettableDoubleObjectInspector)outputOI); - case STRING: - if (outputOI instanceof WritableStringObjectInspector) { - return new PrimitiveObjectInspectorConverter.TextConverter( - (PrimitiveObjectInspector)inputOI); - } else if (outputOI instanceof JavaStringObjectInspector) { - return new PrimitiveObjectInspectorConverter.StringConverter( - (PrimitiveObjectInspector)inputOI); - } - default: - throw new RuntimeException("Hive internal error: conversion of " - + inputOI.getTypeName() + " to " + outputOI.getTypeName() + " not supported yet."); + case PRIMITIVE: + switch (((PrimitiveObjectInspector) outputOI).getPrimitiveCategory()) { + case BOOLEAN: + return new PrimitiveObjectInspectorConverter.BooleanConverter( + (PrimitiveObjectInspector) inputOI, + (SettableBooleanObjectInspector) outputOI); + case BYTE: + return new PrimitiveObjectInspectorConverter.ByteConverter( + (PrimitiveObjectInspector) inputOI, + (SettableByteObjectInspector) outputOI); + case SHORT: + return new PrimitiveObjectInspectorConverter.ShortConverter( + (PrimitiveObjectInspector) inputOI, + (SettableShortObjectInspector) outputOI); + case INT: + return new PrimitiveObjectInspectorConverter.IntConverter( + (PrimitiveObjectInspector) inputOI, + (SettableIntObjectInspector) outputOI); + case LONG: + return new PrimitiveObjectInspectorConverter.LongConverter( + (PrimitiveObjectInspector) inputOI, + (SettableLongObjectInspector) outputOI); + case FLOAT: + return new PrimitiveObjectInspectorConverter.FloatConverter( + (PrimitiveObjectInspector) inputOI, + (SettableFloatObjectInspector) outputOI); + case DOUBLE: + return new PrimitiveObjectInspectorConverter.DoubleConverter( + (PrimitiveObjectInspector) inputOI, + (SettableDoubleObjectInspector) outputOI); + case STRING: + if (outputOI instanceof WritableStringObjectInspector) { + return new PrimitiveObjectInspectorConverter.TextConverter( + (PrimitiveObjectInspector) inputOI); + } else if (outputOI instanceof JavaStringObjectInspector) { + return new PrimitiveObjectInspectorConverter.StringConverter( + (PrimitiveObjectInspector) inputOI); } - case STRUCT: { - return new StructConverter((StructObjectInspector)inputOI, - (SettableStructObjectInspector)outputOI); - } - case LIST: { - return new ListConverter((ListObjectInspector)inputOI, - (SettableListObjectInspector)outputOI); - } - case MAP: { - return new MapConverter((MapObjectInspector)inputOI, - (SettableMapObjectInspector)outputOI); - } default: throw new RuntimeException("Hive internal error: conversion of " - + inputOI.getTypeName() + " to " + outputOI.getTypeName() + " not supported yet."); + + inputOI.getTypeName() + " to " + outputOI.getTypeName() + + " not supported yet."); + } + case STRUCT: { + return new StructConverter((StructObjectInspector) inputOI, + (SettableStructObjectInspector) outputOI); } + case LIST: { + return new ListConverter((ListObjectInspector) inputOI, + (SettableListObjectInspector) outputOI); + } + case MAP: { + return new MapConverter((MapObjectInspector) inputOI, + (SettableMapObjectInspector) outputOI); + } + default: + throw new RuntimeException("Hive internal error: conversion of " + + inputOI.getTypeName() + " to " + outputOI.getTypeName() + + " not supported yet."); + } } - + /** * A converter class for List. */ @@ -127,15 +130,16 @@ ListObjectInspector inputOI; SettableListObjectInspector outputOI; - + ObjectInspector inputElementOI; ObjectInspector outputElementOI; ArrayList elementConverters; - + Object output; - - public ListConverter(ListObjectInspector inputOI, SettableListObjectInspector outputOI) { + + public ListConverter(ListObjectInspector inputOI, + SettableListObjectInspector outputOI) { this.inputOI = inputOI; this.outputOI = outputOI; inputElementOI = inputOI.getListElementObjectInspector(); @@ -143,7 +147,7 @@ output = outputOI.create(0); elementConverters = new ArrayList(); } - + @Override public Object convert(Object input) { if (input == null) { @@ -158,20 +162,20 @@ while (elementConverters.size() < size) { elementConverters.add(getConverter(inputElementOI, outputElementOI)); } - + // Convert the elements outputOI.resize(output, size); for (int index = 0; index < size; index++) { Object inputElement = inputOI.getListElement(input, index); - Object outputElement = elementConverters.get(index).convert(inputElement); + Object outputElement = elementConverters.get(index).convert( + inputElement); outputOI.set(output, index, outputElement); } return output; } - + } - /** * A converter class for Struct. */ @@ -179,44 +183,45 @@ StructObjectInspector inputOI; SettableStructObjectInspector outputOI; - + List inputFields; List outputFields; ArrayList fieldConverters; - + Object output; - - public StructConverter(StructObjectInspector inputOI, SettableStructObjectInspector outputOI) { - + + public StructConverter(StructObjectInspector inputOI, + SettableStructObjectInspector outputOI) { + this.inputOI = inputOI; this.outputOI = outputOI; inputFields = inputOI.getAllStructFieldRefs(); outputFields = outputOI.getAllStructFieldRefs(); - assert(inputFields.size() == outputFields.size()); - + assert (inputFields.size() == outputFields.size()); + fieldConverters = new ArrayList(inputFields.size()); for (int f = 0; f < inputFields.size(); f++) { - fieldConverters.add(getConverter( - inputFields.get(f).getFieldObjectInspector(), - outputFields.get(f).getFieldObjectInspector())); + fieldConverters.add(getConverter(inputFields.get(f) + .getFieldObjectInspector(), outputFields.get(f) + .getFieldObjectInspector())); } output = outputOI.create(); } - + @Override public Object convert(Object input) { if (input == null) { return null; } - + // Convert the fields for (int f = 0; f < inputFields.size(); f++) { - Object inputFieldValue = inputOI.getStructFieldData(input, - inputFields.get(f)); - Object outputFieldValue = fieldConverters.get(f) - .convert(inputFieldValue); - outputOI.setStructFieldData(output, outputFields.get(f), + Object inputFieldValue = inputOI.getStructFieldData(input, inputFields + .get(f)); + Object outputFieldValue = fieldConverters.get(f).convert( + inputFieldValue); + outputOI.setStructFieldData(output, outputFields.get(f), outputFieldValue); } return output; @@ -230,19 +235,20 @@ MapObjectInspector inputOI; SettableMapObjectInspector outputOI; - + ObjectInspector inputKeyOI; ObjectInspector outputKeyOI; ObjectInspector inputValueOI; ObjectInspector outputValueOI; - + ArrayList keyConverters; ArrayList valueConverters; - + Object output; - - public MapConverter(MapObjectInspector inputOI, SettableMapObjectInspector outputOI) { + + public MapConverter(MapObjectInspector inputOI, + SettableMapObjectInspector outputOI) { this.inputOI = inputOI; this.outputOI = outputOI; inputKeyOI = inputOI.getMapKeyObjectInspector(); @@ -253,7 +259,7 @@ valueConverters = new ArrayList(); output = outputOI.create(); } - + @Override public Object convert(Object input) { if (input == null) { @@ -264,37 +270,35 @@ // because the key/valueConverters can reuse the internal object. // So it's not safe to use the same key/valueConverter to convert multiple // key/values. - + // NOTE: This code tries to get all key-value pairs out of the map. // It's not very efficient. The more efficient way should be to let MapOI - // return an Iterator. This is currently not supported by MapOI yet. - + // return an Iterator. This is currently not supported by MapOI yet. + Map map = inputOI.getMap(input); int size = map.size(); - + while (keyConverters.size() < size) { keyConverters.add(getConverter(inputKeyOI, outputKeyOI)); valueConverters.add(getConverter(inputValueOI, outputValueOI)); } - + // CLear the output outputOI.clear(output); - + // Convert the key/value pairs int entryID = 0; - for (Map.Entry entry: map.entrySet()) { + for (Map.Entry entry : map.entrySet()) { Object inputKey = entry.getKey(); Object inputValue = entry.getValue(); Object outputKey = keyConverters.get(entryID).convert(inputKey); Object outputValue = valueConverters.get(entryID).convert(inputValue); - entryID ++; + entryID++; outputOI.put(output, outputKey, outputValue); } return output; } - + } - - } Index: serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StandardListObjectInspector.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StandardListObjectInspector.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StandardListObjectInspector.java (working copy) @@ -20,21 +20,23 @@ import java.util.ArrayList; import java.util.List; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; /** - * DefaultListObjectInspector works on list data that is stored as a Java List or Java Array object. + * DefaultListObjectInspector works on list data that is stored as a Java List + * or Java Array object. * - * Always use the ObjectInspectorFactory to create new ObjectInspector objects, instead - * of directly creating an instance of this class. + * Always use the ObjectInspectorFactory to create new ObjectInspector objects, + * instead of directly creating an instance of this class. */ public class StandardListObjectInspector implements SettableListObjectInspector { ObjectInspector listElementObjectInspector; - - /** Call ObjectInspectorFactory.getStandardListObjectInspector instead. + + /** + * Call ObjectInspectorFactory.getStandardListObjectInspector instead. */ - protected StandardListObjectInspector(ObjectInspector listElementObjectInspector) { + protected StandardListObjectInspector( + ObjectInspector listElementObjectInspector) { this.listElementObjectInspector = listElementObjectInspector; } @@ -46,7 +48,7 @@ public ObjectInspector getListElementObjectInspector() { return listElementObjectInspector; } - + // with data public Object getListElement(Object data, int index) { if (data == null) { @@ -69,7 +71,7 @@ return list.get(index); } } - + public int getListLength(Object data) { if (data == null) { return -1; @@ -85,29 +87,31 @@ return list.size(); } } - + public List getList(Object data) { - if (data == null) return null; + if (data == null) { + return null; + } // We support both List and Object[] // so we have to do differently. if (data.getClass().isArray()) { - data = java.util.Arrays.asList((Object[])data); + data = java.util.Arrays.asList((Object[]) data); } List list = (List) data; return list; } public String getTypeName() { - return org.apache.hadoop.hive.serde.Constants.LIST_TYPE_NAME - + "<" + listElementObjectInspector.getTypeName() + ">"; + return org.apache.hadoop.hive.serde.Constants.LIST_TYPE_NAME + "<" + + listElementObjectInspector.getTypeName() + ">"; } - /////////////////////////////// + // ///////////////////////////// // SettableListObjectInspector @Override public Object create(int size) { List a = new ArrayList(size); - for (int i=0; i a = (List)list; + List a = (List) list; while (a.size() < newSize) { a.add(null); } while (a.size() > newSize) { - a.remove(a.size()-1); + a.remove(a.size() - 1); } return a; } @Override public Object set(Object list, int index, Object element) { - List a = (List)list; + List a = (List) list; a.set(index, element); return a; } Index: serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/UnionStructObjectInspector.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/UnionStructObjectInspector.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/UnionStructObjectInspector.java (working copy) @@ -23,56 +23,61 @@ /** * UnionStructObjectInspector unions several struct data into a single struct. - * Basically, the fields of these structs are put together sequentially into a single struct. + * Basically, the fields of these structs are put together sequentially into a + * single struct. * - * The object that can be acceptable by this ObjectInspector is a List of objects, each of - * which can be inspected by the ObjectInspector provided in the ctor of UnionStructObjectInspector. + * The object that can be acceptable by this ObjectInspector is a List of + * objects, each of which can be inspected by the ObjectInspector provided in + * the ctor of UnionStructObjectInspector. * - * Always use the ObjectInspectorFactory to create new ObjectInspector objects, instead - * of directly creating an instance of this class. + * Always use the ObjectInspectorFactory to create new ObjectInspector objects, + * instead of directly creating an instance of this class. */ public class UnionStructObjectInspector extends StructObjectInspector { public static class MyField implements StructField { public int structID; StructField structField; + public MyField(int structID, StructField structField) { this.structID = structID; this.structField = structField; } + public String getFieldName() { return structField.getFieldName(); } + public ObjectInspector getFieldObjectInspector() { return structField.getFieldObjectInspector(); } } - + List unionObjectInspectors; List fields; - - protected UnionStructObjectInspector(List unionObjectInspectors) { + + protected UnionStructObjectInspector( + List unionObjectInspectors) { init(unionObjectInspectors); } void init(List unionObjectInspectors) { this.unionObjectInspectors = unionObjectInspectors; - + int totalSize = 0; - for (int i=0; i(totalSize); - for (int i=0; i(totalSize); + for (int i = 0; i < unionObjectInspectors.size(); i++) { StructObjectInspector oi = unionObjectInspectors.get(i); - for(StructField sf: oi.getAllStructFieldRefs()) { + for (StructField sf : oi.getAllStructFieldRefs()) { fields.add(new MyField(i, sf)); } } } - - + public final Category getCategory() { return Category.STRUCT; } @@ -82,14 +87,18 @@ } // Without Data + @Override public StructField getStructFieldRef(String fieldName) { return ObjectInspectorUtils.getStandardStructFieldRef(fieldName, fields); } + + @Override public List getAllStructFieldRefs() { return fields; } // With Data + @Override @SuppressWarnings("unchecked") public Object getStructFieldData(Object data, StructField fieldRef) { if (data == null) { @@ -101,15 +110,18 @@ // so we have to do differently. if (data.getClass().isArray()) { Object[] list = (Object[]) data; - assert(list.length == unionObjectInspectors.size()); + assert (list.length == unionObjectInspectors.size()); fieldData = list[f.structID]; } else { List list = (List) data; - assert(list.size() == unionObjectInspectors.size()); + assert (list.size() == unionObjectInspectors.size()); fieldData = list.get(f.structID); } - return unionObjectInspectors.get(f.structID).getStructFieldData(fieldData, f.structField); + return unionObjectInspectors.get(f.structID).getStructFieldData(fieldData, + f.structField); } + + @Override @SuppressWarnings("unchecked") public List getStructFieldsDataAsList(Object data) { if (data == null) { @@ -118,14 +130,15 @@ // We support both List and Object[] // so we have to do differently. if (data.getClass().isArray()) { - data = java.util.Arrays.asList((Object[])data); + data = java.util.Arrays.asList((Object[]) data); } List list = (List) data; - assert(list.size() == unionObjectInspectors.size()); + assert (list.size() == unionObjectInspectors.size()); // Explode ArrayList result = new ArrayList(fields.size()); - for(int i=0; i map = (Map)data; + if (data == null || key == null) { + return null; + } + Map map = (Map) data; return map.get(key); } - + public int getMapSize(Object data) { - if (data == null) return -1; - Map map = (Map)data; + if (data == null) { + return -1; + } + Map map = (Map) data; return map.size(); } - public Map getMap(Object data) { - if (data == null) return null; - Map map = (Map)data; + + public Map getMap(Object data) { + if (data == null) { + return null; + } + Map map = (Map) data; return map; } @@ -77,12 +88,12 @@ } public String getTypeName() { - return org.apache.hadoop.hive.serde.Constants.MAP_TYPE_NAME - + "<" + mapKeyObjectInspector.getTypeName() + "," + return org.apache.hadoop.hive.serde.Constants.MAP_TYPE_NAME + "<" + + mapKeyObjectInspector.getTypeName() + "," + mapValueObjectInspector.getTypeName() + ">"; } - /////////////////////////////// + // ///////////////////////////// // SettableMapObjectInspector @Override public Object create() { @@ -92,21 +103,21 @@ @Override public Object clear(Object map) { - Map m = (HashMap)map; + Map m = (HashMap) map; m.clear(); return m; } @Override public Object put(Object map, Object key, Object value) { - Map m = (HashMap)map; + Map m = (HashMap) map; m.put(key, value); return m; } @Override public Object remove(Object map, Object key) { - Map m = (HashMap)map; + Map m = (HashMap) map; m.remove(key); return m; } Index: serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ReflectionStructObjectInspector.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ReflectionStructObjectInspector.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ReflectionStructObjectInspector.java (working copy) @@ -26,30 +26,35 @@ import org.apache.hadoop.util.ReflectionUtils; /** - * ReflectionStructObjectInspector works on struct data that is stored as a native Java object. - * It will drill down into the Java class to get the fields and construct ObjectInspectors for - * the fields, if they are not specified. + * ReflectionStructObjectInspector works on struct data that is stored as a + * native Java object. It will drill down into the Java class to get the fields + * and construct ObjectInspectors for the fields, if they are not specified. * - * Always use the ObjectInspectorFactory to create new ObjectInspector objects, instead - * of directly creating an instance of this class. - * + * Always use the ObjectInspectorFactory to create new ObjectInspector objects, + * instead of directly creating an instance of this class. + * */ -public class ReflectionStructObjectInspector extends SettableStructObjectInspector { +public class ReflectionStructObjectInspector extends + SettableStructObjectInspector { public static class MyField implements StructField { protected Field field; protected ObjectInspector fieldObjectInspector; - + public MyField(Field field, ObjectInspector fieldObjectInspector) { this.field = field; this.fieldObjectInspector = fieldObjectInspector; } + public String getFieldName() { return field.getName().toLowerCase(); } + public ObjectInspector getFieldObjectInspector() { return fieldObjectInspector; } + + @Override public String toString() { return field.toString(); } @@ -57,7 +62,7 @@ Class objectClass; List fields; - + public Category getCategory() { return Category.STRUCT; } @@ -65,52 +70,60 @@ public String getTypeName() { return objectClass.getName(); } - + /** - * This method is only intended to be used by the Utilities class in this package. - * This creates an uninitialized ObjectInspector so the Utilities class can put it into - * a cache before it initializes when it might look up the cache for member fields that - * might be of the same type (e.g. recursive type like linked list and trees). + * This method is only intended to be used by the Utilities class in this + * package. This creates an uninitialized ObjectInspector so the Utilities + * class can put it into a cache before it initializes when it might look up + * the cache for member fields that might be of the same type (e.g. recursive + * type like linked list and trees). */ ReflectionStructObjectInspector() { } - + /** * This method is only intended to be used by Utilities class in this package. - * The reason that this method is not recursive by itself is because we want to allow - * recursive types. + * The reason that this method is not recursive by itself is because we want + * to allow recursive types. */ - void init(Class objectClass, List structFieldObjectInspectors) { - assert(!List.class.isAssignableFrom(objectClass)); - assert(!Map.class.isAssignableFrom(objectClass)); - + void init(Class objectClass, + List structFieldObjectInspectors) { + assert (!List.class.isAssignableFrom(objectClass)); + assert (!Map.class.isAssignableFrom(objectClass)); + this.objectClass = objectClass; - Field[] reflectionFields = ObjectInspectorUtils.getDeclaredNonStaticFields(objectClass); + Field[] reflectionFields = ObjectInspectorUtils + .getDeclaredNonStaticFields(objectClass); fields = new ArrayList(structFieldObjectInspectors.size()); int used = 0; - for (int i=0; i getAllStructFieldRefs() { return fields; } // With Data + @Override public Object getStructFieldData(Object data, StructField fieldRef) { if (data == null) { return null; @@ -123,22 +136,24 @@ Object r = f.field.get(data); return r; } catch (Exception e) { - throw new RuntimeException("cannot get field " + f.field + " from " - + data.getClass() + " " + data, e); + throw new RuntimeException("cannot get field " + f.field + " from " + + data.getClass() + " " + data, e); } } + + @Override public List getStructFieldsDataAsList(Object data) { if (data == null) { return null; } try { ArrayList result = new ArrayList(fields.size()); - for(int i=0; i fields = soi.getAllStructFieldRefs(); - List fieldNames = new ArrayList(fields.size()); - List fieldObjectInspectors = new ArrayList(fields.size()); - for(StructField f : fields) { - fieldNames.add(f.getFieldName()); - fieldObjectInspectors.add(getStandardObjectInspector(f.getFieldObjectInspector(), objectInspectorOption)); - } - result = ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldObjectInspectors); - break; } - default: { - throw new RuntimeException("Unknown ObjectInspector category!"); + break; + } + case LIST: { + ListObjectInspector loi = (ListObjectInspector) oi; + result = ObjectInspectorFactory + .getStandardListObjectInspector(getStandardObjectInspector(loi + .getListElementObjectInspector(), objectInspectorOption)); + break; + } + case MAP: { + MapObjectInspector moi = (MapObjectInspector) oi; + result = ObjectInspectorFactory.getStandardMapObjectInspector( + getStandardObjectInspector(moi.getMapKeyObjectInspector(), + objectInspectorOption), getStandardObjectInspector(moi + .getMapValueObjectInspector(), objectInspectorOption)); + break; + } + case STRUCT: { + StructObjectInspector soi = (StructObjectInspector) oi; + List fields = soi.getAllStructFieldRefs(); + List fieldNames = new ArrayList(fields.size()); + List fieldObjectInspectors = new ArrayList( + fields.size()); + for (StructField f : fields) { + fieldNames.add(f.getFieldName()); + fieldObjectInspectors.add(getStandardObjectInspector(f + .getFieldObjectInspector(), objectInspectorOption)); } + result = ObjectInspectorFactory.getStandardStructObjectInspector( + fieldNames, fieldObjectInspectors); + break; } + default: { + throw new RuntimeException("Unknown ObjectInspector category!"); + } + } return result; } - + /** * Returns a deep copy of the Object o that can be scanned by a * StandardObjectInspector returned by getStandardObjectInspector(oi). @@ -138,83 +147,88 @@ public static Object copyToStandardObject(Object o, ObjectInspector oi) { return copyToStandardObject(o, oi, ObjectInspectorCopyOption.DEFAULT); } - - public static Object copyToStandardObject(Object o, ObjectInspector oi, ObjectInspectorCopyOption objectInspectorOption) { + + public static Object copyToStandardObject(Object o, ObjectInspector oi, + ObjectInspectorCopyOption objectInspectorOption) { if (o == null) { return null; } - + Object result = null; switch (oi.getCategory()) { - case PRIMITIVE: { - PrimitiveObjectInspector loi = (PrimitiveObjectInspector)oi; - switch (objectInspectorOption) { - case DEFAULT: { - if (loi.preferWritable()) { - result = loi.getPrimitiveWritableObject(loi.copyObject(o)); - } else { - result = loi.getPrimitiveJavaObject(o); - } - break; - } - case JAVA: { - result = loi.getPrimitiveJavaObject(o); - break; - } - case WRITABLE: { - result = loi.getPrimitiveWritableObject(loi.copyObject(o)); - break; - } + case PRIMITIVE: { + PrimitiveObjectInspector loi = (PrimitiveObjectInspector) oi; + switch (objectInspectorOption) { + case DEFAULT: { + if (loi.preferWritable()) { + result = loi.getPrimitiveWritableObject(loi.copyObject(o)); + } else { + result = loi.getPrimitiveJavaObject(o); } break; } - case LIST: { - ListObjectInspector loi = (ListObjectInspector)oi; - int length = loi.getListLength(o); - ArrayList list = new ArrayList(length); - for(int i=0; i map = new HashMap(); - Map omap = moi.getMap(o); - for(Map.Entry entry: omap.entrySet()) { - map.put(copyToStandardObject(entry.getKey(), moi.getMapKeyObjectInspector(), objectInspectorOption), - copyToStandardObject(entry.getValue(), moi.getMapValueObjectInspector(), objectInspectorOption)); - } - result = map; + case WRITABLE: { + result = loi.getPrimitiveWritableObject(loi.copyObject(o)); break; } - case STRUCT: { - StructObjectInspector soi = (StructObjectInspector)oi; - List fields = soi.getAllStructFieldRefs(); - ArrayList struct = new ArrayList(fields.size()); - for(StructField f : fields) { - struct.add(copyToStandardObject(soi.getStructFieldData(o, f), f.getFieldObjectInspector(), objectInspectorOption)); - } - result = struct; - break; } - default: { - throw new RuntimeException("Unknown ObjectInspector category!"); + break; + } + case LIST: { + ListObjectInspector loi = (ListObjectInspector) oi; + int length = loi.getListLength(o); + ArrayList list = new ArrayList(length); + for (int i = 0; i < length; i++) { + list.add(copyToStandardObject(loi.getListElement(o, i), loi + .getListElementObjectInspector(), objectInspectorOption)); } + result = list; + break; } + case MAP: { + MapObjectInspector moi = (MapObjectInspector) oi; + HashMap map = new HashMap(); + Map omap = moi.getMap(o); + for (Map.Entry entry : omap + .entrySet()) { + map.put(copyToStandardObject(entry.getKey(), moi + .getMapKeyObjectInspector(), objectInspectorOption), + copyToStandardObject(entry.getValue(), moi + .getMapValueObjectInspector(), objectInspectorOption)); + } + result = map; + break; + } + case STRUCT: { + StructObjectInspector soi = (StructObjectInspector) oi; + List fields = soi.getAllStructFieldRefs(); + ArrayList struct = new ArrayList(fields.size()); + for (StructField f : fields) { + struct.add(copyToStandardObject(soi.getStructFieldData(o, f), f + .getFieldObjectInspector(), objectInspectorOption)); + } + result = struct; + break; + } + default: { + throw new RuntimeException("Unknown ObjectInspector category!"); + } + } return result; - } - + } + public static String getStandardStructTypeName(StructObjectInspector soi) { StringBuilder sb = new StringBuilder(); sb.append("struct<"); - List fields = soi.getAllStructFieldRefs(); - for(int i=0; i0) sb.append(","); + List fields = soi.getAllStructFieldRefs(); + for (int i = 0; i < fields.size(); i++) { + if (i > 0) { + sb.append(","); + } sb.append(fields.get(i).getFieldName()); sb.append(":"); sb.append(fields.get(i).getFieldObjectInspector().getTypeName()); @@ -222,10 +236,11 @@ sb.append(">"); return sb.toString(); } - - public static StructField getStandardStructFieldRef(String fieldName, List fields) { + + public static StructField getStandardStructFieldRef(String fieldName, + List fields) { fieldName = fieldName.toLowerCase(); - for(int i=0; i=0 && i= 0 && i < fields.size()) { return fields.get(i); } } catch (NumberFormatException e) { // ignore } - throw new RuntimeException("cannot find field " + fieldName + " from " + fields); + throw new RuntimeException("cannot find field " + fieldName + " from " + + fields); // return null; } @@ -260,45 +276,49 @@ } return r; } - + /** - * Get the class names of the ObjectInspector hierarchy. Mainly used for debugging. + * Get the class names of the ObjectInspector hierarchy. Mainly used for + * debugging. */ public static String getObjectInspectorName(ObjectInspector oi) { switch (oi.getCategory()) { - case PRIMITIVE: { - return oi.getClass().getSimpleName(); - } - case LIST: { - ListObjectInspector loi = (ListObjectInspector)oi; - return oi.getClass().getSimpleName() + "<" + getObjectInspectorName(loi.getListElementObjectInspector()) + ">"; - } - case MAP: { - MapObjectInspector moi = (MapObjectInspector)oi; - return oi.getClass().getSimpleName() + "<" + getObjectInspectorName(moi.getMapKeyObjectInspector()) - + "," + getObjectInspectorName(moi.getMapValueObjectInspector()) + ">"; - } - case STRUCT: { - StringBuffer result = new StringBuffer(); - result.append(oi.getClass().getSimpleName() + "<"); - StructObjectInspector soi = (StructObjectInspector)oi; - List fields = soi.getAllStructFieldRefs(); - for(int i = 0; i"); - } else { - result.append(","); - } + case PRIMITIVE: { + return oi.getClass().getSimpleName(); + } + case LIST: { + ListObjectInspector loi = (ListObjectInspector) oi; + return oi.getClass().getSimpleName() + "<" + + getObjectInspectorName(loi.getListElementObjectInspector()) + ">"; + } + case MAP: { + MapObjectInspector moi = (MapObjectInspector) oi; + return oi.getClass().getSimpleName() + "<" + + getObjectInspectorName(moi.getMapKeyObjectInspector()) + "," + + getObjectInspectorName(moi.getMapValueObjectInspector()) + ">"; + } + case STRUCT: { + StringBuffer result = new StringBuffer(); + result.append(oi.getClass().getSimpleName() + "<"); + StructObjectInspector soi = (StructObjectInspector) oi; + List fields = soi.getAllStructFieldRefs(); + for (int i = 0; i < fields.size(); i++) { + result.append(fields.get(i).getFieldName()); + result.append(":"); + result.append(getObjectInspectorName(fields.get(i) + .getFieldObjectInspector())); + if (i == fields.size() - 1) { + result.append(">"); + } else { + result.append(","); } - return result.toString(); } - default: { - throw new RuntimeException("Unknown ObjectInspector category!"); - } + return result.toString(); } + default: { + throw new RuntimeException("Unknown ObjectInspector category!"); + } + } } public static int hashCode(Object o, ObjectInspector objIns) { @@ -306,58 +326,68 @@ return 0; } switch (objIns.getCategory()) { - case PRIMITIVE: { - PrimitiveObjectInspector poi = ((PrimitiveObjectInspector)objIns); - switch (poi.getPrimitiveCategory()) { - case VOID: return 0; - case BOOLEAN: return ((BooleanObjectInspector)poi).get(o) ? 1 : 0; - case BYTE: return ((ByteObjectInspector)poi).get(o); - case SHORT: return ((ShortObjectInspector)poi).get(o); - case INT: return ((IntObjectInspector)poi).get(o); - case LONG: { - long a = ((LongObjectInspector)poi).get(o); - return (int)((a >>> 32) ^ a); - } - case FLOAT: return Float.floatToIntBits(((FloatObjectInspector)poi).get(o)); - case DOUBLE: { - // This hash function returns the same result as Double.hashCode() - // while DoubleWritable.hashCode returns a different result. - long a = Double.doubleToLongBits(((DoubleObjectInspector)poi).get(o)); - return (int)((a >>> 32) ^ a); - } - case STRING: { - // This hash function returns the same result as String.hashCode() when - // all characters are ASCII, while Text.hashCode() always returns a - // different result. - Text t = ((StringObjectInspector)poi).getPrimitiveWritableObject(o); - int r = 0; - for (int i=0; i>> 32) ^ a); + } + case FLOAT: + return Float.floatToIntBits(((FloatObjectInspector) poi).get(o)); + case DOUBLE: { + // This hash function returns the same result as Double.hashCode() + // while DoubleWritable.hashCode returns a different result. + long a = Double.doubleToLongBits(((DoubleObjectInspector) poi).get(o)); + return (int) ((a >>> 32) ^ a); + } + case STRING: { + // This hash function returns the same result as String.hashCode() when + // all characters are ASCII, while Text.hashCode() always returns a + // different result. + Text t = ((StringObjectInspector) poi).getPrimitiveWritableObject(o); + int r = 0; + for (int i = 0; i < t.getLength(); i++) { + r = r * 31 + t.getBytes()[i]; } + return r; } - case STRUCT: - case LIST: - case MAP: - default: - throw new RuntimeException("Hash code on complex types not supported yet."); + default: { + throw new RuntimeException("Unknown type: " + + poi.getPrimitiveCategory()); + } + } } + case STRUCT: + case LIST: + case MAP: + default: + throw new RuntimeException( + "Hash code on complex types not supported yet."); + } } /** - * Compare two arrays of objects with their respective arrays of ObjectInspectors. + * Compare two arrays of objects with their respective arrays of + * ObjectInspectors. */ - public static int compare(Object[] o1, ObjectInspector[] oi1, Object[] o2, ObjectInspector[] oi2) { - assert(o1.length == oi1.length); - assert(o2.length == oi2.length); - assert(o1.length == o2.length); - - for (int i=0; i v2 ? 1 : (v1 < v2 ? -1 : 0); + } + case LONG: { + long v1 = ((LongObjectInspector) poi1).get(o1); + long v2 = ((LongObjectInspector) poi2).get(o2); + return v1 > v2 ? 1 : (v1 < v2 ? -1 : 0); + } + case FLOAT: { + float v1 = ((FloatObjectInspector) poi1).get(o1); + float v2 = ((FloatObjectInspector) poi2).get(o2); + return v1 > v2 ? 1 : (v1 < v2 ? -1 : 0); + } + case DOUBLE: { + double v1 = ((DoubleObjectInspector) poi1).get(o1); + double v2 = ((DoubleObjectInspector) poi2).get(o2); + return v1 > v2 ? 1 : (v1 < v2 ? -1 : 0); + } + case STRING: { + if (poi1.preferWritable() || poi2.preferWritable()) { + Text t1 = (Text) poi1.getPrimitiveWritableObject(o1); + Text t2 = (Text) poi2.getPrimitiveWritableObject(o2); + return t1 == null ? (t2 == null ? 0 : -1) : (t2 == null ? 1 + : ShimLoader.getHadoopShims().compareText(t1, t2)); + } else { + String s1 = (String) poi1.getPrimitiveJavaObject(o1); + String s2 = (String) poi2.getPrimitiveJavaObject(o2); + return s1 == null ? (s2 == null ? 0 : -1) : (s2 == null ? 1 : s1 + .compareTo(s2)); } - switch (poi1.getPrimitiveCategory()) { - case VOID: return 0; - case BOOLEAN: { - int v1 = ((BooleanObjectInspector)poi1).get(o1) ? 1 : 0; - int v2 = ((BooleanObjectInspector)poi2).get(o2) ? 1 : 0; - return v1 - v2; - } - case BYTE: { - int v1 = ((ByteObjectInspector)poi1).get(o1); - int v2 = ((ByteObjectInspector)poi2).get(o2); - return v1 - v2; - } - case SHORT: { - int v1 = ((ShortObjectInspector)poi1).get(o1); - int v2 = ((ShortObjectInspector)poi2).get(o2); - return v1 - v2; - } - case INT: { - int v1 = ((IntObjectInspector)poi1).get(o1); - int v2 = ((IntObjectInspector)poi2).get(o2); - return v1 > v2 ? 1 : (v1 < v2 ? -1 : 0); - } - case LONG: { - long v1 = ((LongObjectInspector)poi1).get(o1); - long v2 = ((LongObjectInspector)poi2).get(o2); - return v1 > v2 ? 1 : (v1 < v2 ? -1 : 0); - } - case FLOAT: { - float v1 = ((FloatObjectInspector)poi1).get(o1); - float v2 = ((FloatObjectInspector)poi2).get(o2); - return v1 > v2 ? 1 : (v1 < v2 ? -1 : 0); - } - case DOUBLE: { - double v1 = ((DoubleObjectInspector)poi1).get(o1); - double v2 = ((DoubleObjectInspector)poi2).get(o2); - return v1 > v2 ? 1 : (v1 < v2 ? -1 : 0); - } - case STRING: { - if (poi1.preferWritable() || poi2.preferWritable()) { - Text t1 = (Text)poi1.getPrimitiveWritableObject(o1); - Text t2 = (Text)poi2.getPrimitiveWritableObject(o2); - return t1 == null - ? (t2 == null ? 0 : -1) - : (t2 == null ? 1 : - ShimLoader.getHadoopShims().compareText(t1, t2)); - } else { - String s1 = (String)poi1.getPrimitiveJavaObject(o1); - String s2 = (String)poi2.getPrimitiveJavaObject(o2); - return s1 == null - ? (s2 == null ? 0 : -1) - : (s2 == null ? 1 : s1.compareTo(s2)); - } - } - default: { - throw new RuntimeException("Unknown type: " + poi1.getPrimitiveCategory()); - } - } } - case STRUCT: { - StructObjectInspector soi1 = (StructObjectInspector)oi1; - StructObjectInspector soi2 = (StructObjectInspector)oi2; - List fields1 = soi1.getAllStructFieldRefs(); - List fields2 = soi2.getAllStructFieldRefs(); - int minimum = Math.min(fields1.size(), fields2.size()); - for (int i=0; i fields1 = soi1.getAllStructFieldRefs(); + List fields2 = soi2.getAllStructFieldRefs(); + int minimum = Math.min(fields1.size(), fields2.size()); + for (int i = 0; i < minimum; i++) { + int r = compare(soi1.getStructFieldData(o1, fields1.get(i)), fields1 + .get(i).getFieldObjectInspector(), soi2.getStructFieldData(o2, + fields2.get(i)), fields2.get(i).getFieldObjectInspector()); + if (r != 0) { + return r; } - return fields1.size() - fields2.size(); } - case LIST: { - ListObjectInspector loi1 = (ListObjectInspector)oi1; - ListObjectInspector loi2 = (ListObjectInspector)oi2; - int minimum = Math.min(loi1.getListLength(o1), loi2.getListLength(o2)); - for (int i=0; i fields = soi.getAllStructFieldRefs(); StringBuilder sb = new StringBuilder(); - for (int i=0; i 0) { sb.append(","); } @@ -504,16 +536,14 @@ public static String getFieldTypes(StructObjectInspector soi) { List fields = soi.getAllStructFieldRefs(); StringBuilder sb = new StringBuilder(); - for (int i=0; i 0) { sb.append(":"); } - sb.append( - TypeInfoUtils.getTypeInfoFromObjectInspector( - fields.get(i).getFieldObjectInspector()).getTypeName()); + sb.append(TypeInfoUtils.getTypeInfoFromObjectInspector( + fields.get(i).getFieldObjectInspector()).getTypeName()); } return sb.toString(); } - } Index: serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/SettableMapObjectInspector.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/SettableMapObjectInspector.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/SettableMapObjectInspector.java (working copy) @@ -23,17 +23,17 @@ * Create an empty map. */ public Object create(); - + /** * Add a key-value pair to the map. Return the map. */ public Object put(Object map, Object key, Object value); - + /** * Remove a key-value pair from the map. Return the map. */ public Object remove(Object map, Object key); - + /** * Clear the map. Return the map. */ Index: serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/InspectableObject.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/InspectableObject.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/InspectableObject.java (working copy) @@ -21,24 +21,24 @@ /** * Simple wrapper of object with ObjectInspector. * - * TODO: we need to redefine the hashCode and equals methods, so that - * it can be put into a HashMap as a key. + * TODO: we need to redefine the hashCode and equals methods, so that it can be + * put into a HashMap as a key. * - * This class also serves as a facility for a function that returns - * both an object and an ObjectInspector. + * This class also serves as a facility for a function that returns both an + * object and an ObjectInspector. */ public class InspectableObject { public Object o; public ObjectInspector oi; - + public InspectableObject() { - this(null,null); + this(null, null); } + public InspectableObject(Object o, ObjectInspector oi) { this.o = o; this.oi = oi; } - } Index: serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ThriftStructObjectInspector.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ThriftStructObjectInspector.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ThriftStructObjectInspector.java (working copy) @@ -20,14 +20,14 @@ /** * - * Always use the ObjectInspectorFactory to create new ObjectInspector objects, instead - * of directly creating an instance of this class. + * Always use the ObjectInspectorFactory to create new ObjectInspector objects, + * instead of directly creating an instance of this class. */ -class ThriftStructObjectInspector extends - ReflectionStructObjectInspector { +class ThriftStructObjectInspector extends ReflectionStructObjectInspector { + @Override public boolean shouldIgnoreField(String name) { return name.startsWith("__isset"); } - + } Index: serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StandardStructObjectInspector.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StandardStructObjectInspector.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StandardStructObjectInspector.java (working copy) @@ -26,25 +26,29 @@ import org.apache.commons.logging.LogFactory; /** - * ListStructObjectInspector works on struct data that is stored as a Java List or Java Array object. - * Basically, the fields are stored sequentially in the List object. + * ListStructObjectInspector works on struct data that is stored as a Java List + * or Java Array object. Basically, the fields are stored sequentially in the + * List object. * - * The names of the struct fields and the internal structure of the struct fields are specified in - * the ctor of the StructObjectInspector. + * The names of the struct fields and the internal structure of the struct + * fields are specified in the ctor of the StructObjectInspector. * - * Always use the ObjectInspectorFactory to create new ObjectInspector objects, instead - * of directly creating an instance of this class. + * Always use the ObjectInspectorFactory to create new ObjectInspector objects, + * instead of directly creating an instance of this class. */ -public class StandardStructObjectInspector extends SettableStructObjectInspector { +public class StandardStructObjectInspector extends + SettableStructObjectInspector { - public static final Log LOG = LogFactory.getLog(StandardStructObjectInspector.class.getName()); - + public static final Log LOG = LogFactory + .getLog(StandardStructObjectInspector.class.getName()); + protected static class MyField implements StructField { protected int fieldID; protected String fieldName; protected ObjectInspector fieldObjectInspector; - - public MyField(int fieldID, String fieldName, ObjectInspector fieldObjectInspector) { + + public MyField(int fieldID, String fieldName, + ObjectInspector fieldObjectInspector) { this.fieldID = fieldID; this.fieldName = fieldName.toLowerCase(); this.fieldObjectInspector = fieldObjectInspector; @@ -53,63 +57,77 @@ public int getFieldID() { return fieldID; } + public String getFieldName() { return fieldName; } + public ObjectInspector getFieldObjectInspector() { return fieldObjectInspector; } - + + @Override public String toString() { return "" + fieldID + ":" + fieldName; } } - + protected List fields; - + public String getTypeName() { return ObjectInspectorUtils.getStandardStructTypeName(this); } - - /** Call ObjectInspectorFactory.getStandardListObjectInspector instead. + + /** + * Call ObjectInspectorFactory.getStandardListObjectInspector instead. */ - protected StandardStructObjectInspector(List structFieldNames, List structFieldObjectInspectors) { + protected StandardStructObjectInspector(List structFieldNames, + List structFieldObjectInspectors) { init(structFieldNames, structFieldObjectInspectors); } - protected void init(List structFieldNames, List structFieldObjectInspectors) { - assert(structFieldNames.size() == structFieldObjectInspectors.size()); - - fields = new ArrayList(structFieldNames.size()); - for(int i=0; i structFieldNames, + List structFieldObjectInspectors) { + assert (structFieldNames.size() == structFieldObjectInspectors.size()); + + fields = new ArrayList(structFieldNames.size()); + for (int i = 0; i < structFieldNames.size(); i++) { + fields.add(new MyField(i, structFieldNames.get(i), + structFieldObjectInspectors.get(i))); } } - + protected StandardStructObjectInspector(List fields) { init(fields); } + protected void init(List fields) { - this.fields = new ArrayList(fields.size()); - for(int i=0; i(fields.size()); + for (int i = 0; i < fields.size(); i++) { + this.fields.add(new MyField(i, fields.get(i).getFieldName(), fields + .get(i).getFieldObjectInspector())); } } - public final Category getCategory() { return Category.STRUCT; } // Without Data + @Override public StructField getStructFieldRef(String fieldName) { return ObjectInspectorUtils.getStandardStructFieldRef(fieldName, fields); } + + @Override public List getAllStructFieldRefs() { return fields; } boolean warned = false; + // With Data + @Override @SuppressWarnings("unchecked") public Object getStructFieldData(Object data, StructField fieldRef) { if (data == null) { @@ -118,32 +136,33 @@ // We support both List and Object[] // so we have to do differently. boolean isArray = data.getClass().isArray(); - if(!isArray && !(data instanceof List)) { + if (!isArray && !(data instanceof List)) { return data; } - int listSize = (isArray - ? ((Object[])data).length - : ((List)data).size()); + int listSize = (isArray ? ((Object[]) data).length : ((List) data) + .size()); MyField f = (MyField) fieldRef; if (fields.size() != listSize && !warned) { // TODO: remove this warned = true; - LOG.warn("Trying to access " + fields.size() + " fields inside a list of " - + listSize + " elements: " - + (isArray ? Arrays.asList((Object[])data) : (List)data)); + LOG.warn("Trying to access " + fields.size() + + " fields inside a list of " + listSize + " elements: " + + (isArray ? Arrays.asList((Object[]) data) : (List) data)); LOG.warn("ignoring similar errors."); } int fieldID = f.getFieldID(); - assert(fieldID >= 0 && fieldID < fields.size()); + assert (fieldID >= 0 && fieldID < fields.size()); if (fieldID >= listSize) { return null; } else if (isArray) { - return ((Object[])data)[fieldID]; + return ((Object[]) data)[fieldID]; } else { - return ((List)data).get(fieldID); + return ((List) data).get(fieldID); } } + + @Override @SuppressWarnings("unchecked") public List getStructFieldsDataAsList(Object data) { if (data == null) { @@ -152,30 +171,31 @@ // We support both List and Object[] // so we have to do differently. if (data.getClass().isArray()) { - data = java.util.Arrays.asList((Object[])data); + data = java.util.Arrays.asList((Object[]) data); } List list = (List) data; - assert(list.size() == fields.size()); + assert (list.size() == fields.size()); return list; } - /////////////////////////////// + // ///////////////////////////// // SettableStructObjectInspector @Override public Object create() { ArrayList a = new ArrayList(fields.size()); - for (int i=0; i a = (ArrayList)struct; - MyField myField = (MyField)field; + public Object setStructFieldData(Object struct, StructField field, + Object fieldValue) { + ArrayList a = (ArrayList) struct; + MyField myField = (MyField) field; a.set(myField.fieldID, fieldValue); return a; } - + } Index: serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorFactory.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorFactory.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorFactory.java (working copy) @@ -20,7 +20,6 @@ import java.lang.reflect.Field; import java.lang.reflect.GenericArrayType; -import java.lang.reflect.Modifier; import java.lang.reflect.ParameterizedType; import java.lang.reflect.Type; import java.util.ArrayList; @@ -28,137 +27,152 @@ import java.util.List; import java.util.Map; -import javax.swing.event.ListSelectionEvent; - import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; import org.apache.hadoop.io.Text; - /** - * ObjectInspectorFactory is the primary way to create new ObjectInspector instances. + * ObjectInspectorFactory is the primary way to create new ObjectInspector + * instances. * - * SerDe classes should call the static functions in this library to create an ObjectInspector - * to return to the caller of SerDe2.getObjectInspector(). + * SerDe classes should call the static functions in this library to create an + * ObjectInspector to return to the caller of SerDe2.getObjectInspector(). * - * The reason of having caches here is that ObjectInspector is because ObjectInspectors do - * not have an internal state - so ObjectInspectors with the same construction parameters should - * result in exactly the same ObjectInspector. + * The reason of having caches here is that ObjectInspector is because + * ObjectInspectors do not have an internal state - so ObjectInspectors with the + * same construction parameters should result in exactly the same + * ObjectInspector. */ public class ObjectInspectorFactory { - /** - * ObjectInspectorOptions describes what ObjectInspector to use. - * JAVA is to use pure JAVA reflection. THRIFT is to use JAVA reflection and filter out __isset fields. - * New ObjectInspectorOptions can be added here when available. + * ObjectInspectorOptions describes what ObjectInspector to use. JAVA is to + * use pure JAVA reflection. THRIFT is to use JAVA reflection and filter out + * __isset fields. New ObjectInspectorOptions can be added here when + * available. * - * We choose to use a single HashMap objectInspectorCache to cache all situations for efficiency and code - * simplicity. And we don't expect a case that a user need to create 2 or more different types of - * ObjectInspectors for the same Java type. + * We choose to use a single HashMap objectInspectorCache to cache all + * situations for efficiency and code simplicity. And we don't expect a case + * that a user need to create 2 or more different types of ObjectInspectors + * for the same Java type. */ public enum ObjectInspectorOptions { - JAVA, - THRIFT + JAVA, THRIFT }; - + private static HashMap objectInspectorCache = new HashMap(); - - public static ObjectInspector getReflectionObjectInspector(Type t, ObjectInspectorOptions options) { + + public static ObjectInspector getReflectionObjectInspector(Type t, + ObjectInspectorOptions options) { ObjectInspector oi = objectInspectorCache.get(t); if (oi == null) { oi = getReflectionObjectInspectorNoCache(t, options); objectInspectorCache.put(t, oi); } - if ((options.equals(ObjectInspectorOptions.JAVA) && oi.getClass().equals(ThriftStructObjectInspector.class)) - || (options.equals(ObjectInspectorOptions.THRIFT) && oi.getClass().equals(ReflectionStructObjectInspector.class))) { - throw new RuntimeException("Cannot call getObjectInspectorByReflection with both JAVA and THRIFT !"); + if ((options.equals(ObjectInspectorOptions.JAVA) && oi.getClass().equals( + ThriftStructObjectInspector.class)) + || (options.equals(ObjectInspectorOptions.THRIFT) && oi.getClass() + .equals(ReflectionStructObjectInspector.class))) { + throw new RuntimeException( + "Cannot call getObjectInspectorByReflection with both JAVA and THRIFT !"); } return oi; } - - private static ObjectInspector getReflectionObjectInspectorNoCache(Type t, ObjectInspectorOptions options) { + + private static ObjectInspector getReflectionObjectInspectorNoCache(Type t, + ObjectInspectorOptions options) { if (t instanceof GenericArrayType) { - GenericArrayType at = (GenericArrayType)t; - return getStandardListObjectInspector( - getReflectionObjectInspector(at.getGenericComponentType(), options)); + GenericArrayType at = (GenericArrayType) t; + return getStandardListObjectInspector(getReflectionObjectInspector(at + .getGenericComponentType(), options)); } if (t instanceof ParameterizedType) { - ParameterizedType pt = (ParameterizedType)t; + ParameterizedType pt = (ParameterizedType) t; // List? - if (List.class.isAssignableFrom((Class)pt.getRawType())) { - return getStandardListObjectInspector( - getReflectionObjectInspector(pt.getActualTypeArguments()[0], options)); + if (List.class.isAssignableFrom((Class) pt.getRawType())) { + return getStandardListObjectInspector(getReflectionObjectInspector(pt + .getActualTypeArguments()[0], options)); } // Map? - if (Map.class.isAssignableFrom((Class)pt.getRawType())) { - return getStandardMapObjectInspector( - getReflectionObjectInspector(pt.getActualTypeArguments()[0], options), - getReflectionObjectInspector(pt.getActualTypeArguments()[1], options)); + if (Map.class.isAssignableFrom((Class) pt.getRawType())) { + return getStandardMapObjectInspector(getReflectionObjectInspector(pt + .getActualTypeArguments()[0], options), + getReflectionObjectInspector(pt.getActualTypeArguments()[1], + options)); } - // Otherwise convert t to RawType so we will fall into the following if block. + // Otherwise convert t to RawType so we will fall into the following if + // block. t = pt.getRawType(); } - + // Must be a class. if (!(t instanceof Class)) { - throw new RuntimeException(ObjectInspectorFactory.class.getName() + " internal error:" - + t); + throw new RuntimeException(ObjectInspectorFactory.class.getName() + + " internal error:" + t); } - Class c = (Class)t; - + Class c = (Class) t; + // Java Primitive Type? if (PrimitiveObjectInspectorUtils.isPrimitiveJavaType(c)) { - return PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector( - PrimitiveObjectInspectorUtils.getTypeEntryFromPrimitiveJavaType(c).primitiveCategory); + return PrimitiveObjectInspectorFactory + .getPrimitiveJavaObjectInspector(PrimitiveObjectInspectorUtils + .getTypeEntryFromPrimitiveJavaType(c).primitiveCategory); } // Java Primitive Class? if (PrimitiveObjectInspectorUtils.isPrimitiveJavaClass(c)) { - return PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector( - PrimitiveObjectInspectorUtils.getTypeEntryFromPrimitiveJavaClass(c).primitiveCategory); + return PrimitiveObjectInspectorFactory + .getPrimitiveJavaObjectInspector(PrimitiveObjectInspectorUtils + .getTypeEntryFromPrimitiveJavaClass(c).primitiveCategory); } - + // Primitive Writable class? if (PrimitiveObjectInspectorUtils.isPrimitiveWritableClass(c)) { - return PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector( - PrimitiveObjectInspectorUtils.getTypeEntryFromPrimitiveWritableClass(c).primitiveCategory); + return PrimitiveObjectInspectorFactory + .getPrimitiveWritableObjectInspector(PrimitiveObjectInspectorUtils + .getTypeEntryFromPrimitiveWritableClass(c).primitiveCategory); } // Must be struct because List and Map need to be ParameterizedType - assert(!List.class.isAssignableFrom(c)); - assert(!Map.class.isAssignableFrom(c)); - + assert (!List.class.isAssignableFrom(c)); + assert (!Map.class.isAssignableFrom(c)); + // Create StructObjectInspector ReflectionStructObjectInspector oi; - switch(options) { - case JAVA: + switch (options) { + case JAVA: oi = new ReflectionStructObjectInspector(); break; - case THRIFT: + case THRIFT: oi = new ThriftStructObjectInspector(); break; default: - throw new RuntimeException(ObjectInspectorFactory.class.getName() + ": internal error."); + throw new RuntimeException(ObjectInspectorFactory.class.getName() + + ": internal error."); } - // put it into the cache BEFORE it is initialized to make sure we can catch recursive types. + // put it into the cache BEFORE it is initialized to make sure we can catch + // recursive types. objectInspectorCache.put(t, oi); Field[] fields = ObjectInspectorUtils.getDeclaredNonStaticFields(c); - ArrayList structFieldObjectInspectors = new ArrayList(fields.length); - for(int i=0; i structFieldObjectInspectors = new ArrayList( + fields.length); + for (int i = 0; i < fields.length; i++) { if (!oi.shouldIgnoreField(fields[i].getName())) { - structFieldObjectInspectors.add(getReflectionObjectInspector(fields[i].getGenericType(), options)); + structFieldObjectInspectors.add(getReflectionObjectInspector(fields[i] + .getGenericType(), options)); } } oi.init(c, structFieldObjectInspectors); return oi; } - - static HashMap cachedStandardListObjectInspector = - new HashMap(); - public static StandardListObjectInspector getStandardListObjectInspector(ObjectInspector listElementObjectInspector) { - StandardListObjectInspector result = cachedStandardListObjectInspector.get(listElementObjectInspector); + + static HashMap cachedStandardListObjectInspector = new HashMap(); + + public static StandardListObjectInspector getStandardListObjectInspector( + ObjectInspector listElementObjectInspector) { + StandardListObjectInspector result = cachedStandardListObjectInspector + .get(listElementObjectInspector); if (result == null) { result = new StandardListObjectInspector(listElementObjectInspector); cachedStandardListObjectInspector.put(listElementObjectInspector, result); @@ -166,46 +180,54 @@ return result; } - static HashMap, StandardMapObjectInspector> cachedStandardMapObjectInspector = - new HashMap, StandardMapObjectInspector>(); - public static StandardMapObjectInspector getStandardMapObjectInspector(ObjectInspector mapKeyObjectInspector, ObjectInspector mapValueObjectInspector) { + static HashMap, StandardMapObjectInspector> cachedStandardMapObjectInspector = new HashMap, StandardMapObjectInspector>(); + + public static StandardMapObjectInspector getStandardMapObjectInspector( + ObjectInspector mapKeyObjectInspector, + ObjectInspector mapValueObjectInspector) { ArrayList signature = new ArrayList(2); signature.add(mapKeyObjectInspector); signature.add(mapValueObjectInspector); - StandardMapObjectInspector result = cachedStandardMapObjectInspector.get(signature); + StandardMapObjectInspector result = cachedStandardMapObjectInspector + .get(signature); if (result == null) { - result = new StandardMapObjectInspector(mapKeyObjectInspector, mapValueObjectInspector); + result = new StandardMapObjectInspector(mapKeyObjectInspector, + mapValueObjectInspector); cachedStandardMapObjectInspector.put(signature, result); } return result; } - - static HashMap>, StandardStructObjectInspector> cachedStandardStructObjectInspector = - new HashMap>, StandardStructObjectInspector>(); - public static StandardStructObjectInspector getStandardStructObjectInspector(List structFieldNames, List structFieldObjectInspectors) { + + static HashMap>, StandardStructObjectInspector> cachedStandardStructObjectInspector = new HashMap>, StandardStructObjectInspector>(); + + public static StandardStructObjectInspector getStandardStructObjectInspector( + List structFieldNames, + List structFieldObjectInspectors) { ArrayList> signature = new ArrayList>(); signature.add(structFieldNames); signature.add(structFieldObjectInspectors); - StandardStructObjectInspector result = cachedStandardStructObjectInspector.get(signature); + StandardStructObjectInspector result = cachedStandardStructObjectInspector + .get(signature); if (result == null) { - result = new StandardStructObjectInspector(structFieldNames, structFieldObjectInspectors); + result = new StandardStructObjectInspector(structFieldNames, + structFieldObjectInspectors); cachedStandardStructObjectInspector.put(signature, result); } return result; } - - static HashMap, UnionStructObjectInspector> cachedUnionStructObjectInspector = - new HashMap, UnionStructObjectInspector>(); - public static UnionStructObjectInspector getUnionStructObjectInspector(List structObjectInspectors) { - UnionStructObjectInspector result = cachedUnionStructObjectInspector.get(structObjectInspectors); + static HashMap, UnionStructObjectInspector> cachedUnionStructObjectInspector = new HashMap, UnionStructObjectInspector>(); + + public static UnionStructObjectInspector getUnionStructObjectInspector( + List structObjectInspectors) { + UnionStructObjectInspector result = cachedUnionStructObjectInspector + .get(structObjectInspectors); if (result == null) { result = new UnionStructObjectInspector(structObjectInspectors); cachedUnionStructObjectInspector.put(structObjectInspectors, result); } return result; } - static HashMap, ColumnarStructObjectInspector> cachedColumnarStructObjectInspector = new HashMap, ColumnarStructObjectInspector>(); @@ -225,5 +247,5 @@ } return result; } - + } Index: serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/SettableStructObjectInspector.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/SettableStructObjectInspector.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/SettableStructObjectInspector.java (working copy) @@ -17,15 +17,17 @@ */ package org.apache.hadoop.hive.serde2.objectinspector; -public abstract class SettableStructObjectInspector extends StructObjectInspector { - +public abstract class SettableStructObjectInspector extends + StructObjectInspector { + /** * Create a struct which is not null, but all its fields are null. */ public abstract Object create(); - + /** - * Modify the value of a field. Returns the struct. + * Modify the value of a field. Returns the struct. */ - public abstract Object setStructFieldData(Object struct, StructField field, Object fieldValue); + public abstract Object setStructFieldData(Object struct, StructField field, + Object fieldValue); } Index: serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/PrimitiveObjectInspector.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/PrimitiveObjectInspector.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/PrimitiveObjectInspector.java (working copy) @@ -20,31 +20,31 @@ public interface PrimitiveObjectInspector extends ObjectInspector { /** - * The primitive types supported by Hive. + * The primitive types supported by Hive. */ public static enum PrimitiveCategory { VOID, BOOLEAN, BYTE, SHORT, INT, LONG, FLOAT, DOUBLE, STRING, UNKNOWN }; - + /** * Get the primitive category of the PrimitiveObjectInspector. */ public PrimitiveCategory getPrimitiveCategory(); - + /** - * Get the Primitive Writable class which is the return type of + * Get the Primitive Writable class which is the return type of * getPrimitiveWritableObject() and copyToPrimitiveWritableObject() */ public Class getPrimitiveWritableClass(); /** - * Return the data in an instance of primitive writable Object. If the - * Object is already a primitive writable Object, just return o. + * Return the data in an instance of primitive writable Object. If the Object + * is already a primitive writable Object, just return o. */ public Object getPrimitiveWritableObject(Object o); - + /** - * Get the Java Primitive class which is the return type of + * Get the Java Primitive class which is the return type of * getJavaPrimitiveObject(). */ public Class getJavaPrimitiveClass(); @@ -53,21 +53,20 @@ * Get the Java Primitive object. */ public Object getPrimitiveJavaObject(Object o); - + /** - * Get a copy of the Object in the same class, so the return value can be + * Get a copy of the Object in the same class, so the return value can be * stored independently of the parameter. * * If the Object is a Primitive Java Object, we just return the parameter * since Primitive Java Object is immutable. */ public Object copyObject(Object o); - + /** * Whether the ObjectInspector prefers to return a Primitive Writable Object - * instead of a Primitive Java Object. - * This can be useful for determining the most efficient way to getting - * data out of the Object. + * instead of a Primitive Java Object. This can be useful for determining the + * most efficient way to getting data out of the Object. */ public boolean preferWritable(); } Index: serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/JavaBooleanObjectInspector.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/JavaBooleanObjectInspector.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/JavaBooleanObjectInspector.java (working copy) @@ -19,25 +19,25 @@ import org.apache.hadoop.io.BooleanWritable; - /** * A JavaBooleanObjectInspector inspects a Java Boolean Object. */ -public class JavaBooleanObjectInspector extends AbstractPrimitiveJavaObjectInspector -implements SettableBooleanObjectInspector{ +public class JavaBooleanObjectInspector extends + AbstractPrimitiveJavaObjectInspector implements + SettableBooleanObjectInspector { JavaBooleanObjectInspector() { super(PrimitiveObjectInspectorUtils.booleanTypeEntry); } - + @Override public Object getPrimitiveWritableObject(Object o) { - return o == null ? null : new BooleanWritable(((Boolean)o).booleanValue()); + return o == null ? null : new BooleanWritable(((Boolean) o).booleanValue()); } @Override public boolean get(Object o) { - return ((Boolean)o).booleanValue(); + return ((Boolean) o).booleanValue(); } @Override Index: serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/SettableShortObjectInspector.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/SettableShortObjectInspector.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/SettableShortObjectInspector.java (working copy) @@ -17,7 +17,6 @@ */ package org.apache.hadoop.hive.serde2.objectinspector.primitive; - /** * A SettableShortObjectInspector can set a short value to an object. */ @@ -26,11 +25,11 @@ /** * Set the object with the value. Return the object that has the new value. * - * In most cases the returned value should be the same as o, but in case - * o is unmodifiable, this will return a new object with new value. + * In most cases the returned value should be the same as o, but in case o is + * unmodifiable, this will return a new object with new value. */ public Object set(Object o, short value); - + /** * Create an object with the value. */ Index: serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/SettableLongObjectInspector.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/SettableLongObjectInspector.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/SettableLongObjectInspector.java (working copy) @@ -17,7 +17,6 @@ */ package org.apache.hadoop.hive.serde2.objectinspector.primitive; - /** * A SettableLongObjectInspector can set a long value to an object. */ @@ -26,11 +25,11 @@ /** * Set the object with the value. Return the object that has the new value. * - * In most cases the returned value should be the same as o, but in case - * o is unmodifiable, this will return a new object with new value. + * In most cases the returned value should be the same as o, but in case o is + * unmodifiable, this will return a new object with new value. */ public Object set(Object o, long value); - + /** * Create an object with the value. */ Index: serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableIntObjectInspector.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableIntObjectInspector.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableIntObjectInspector.java (working copy) @@ -17,35 +17,34 @@ */ package org.apache.hadoop.hive.serde2.objectinspector.primitive; -import org.apache.hadoop.io.FloatWritable; import org.apache.hadoop.io.IntWritable; - /** * A WritableIntObjectInspector inspects a IntWritable Object. */ -public class WritableIntObjectInspector extends AbstractPrimitiveWritableObjectInspector -implements SettableIntObjectInspector{ +public class WritableIntObjectInspector extends + AbstractPrimitiveWritableObjectInspector implements + SettableIntObjectInspector { WritableIntObjectInspector() { super(PrimitiveObjectInspectorUtils.intTypeEntry); } - + @Override public int get(Object o) { - return ((IntWritable)o).get(); + return ((IntWritable) o).get(); } @Override public Object copyObject(Object o) { - return o == null ? null : new IntWritable(((IntWritable)o).get()); + return o == null ? null : new IntWritable(((IntWritable) o).get()); } @Override public Object getPrimitiveJavaObject(Object o) { - return o == null ? null : Integer.valueOf(((IntWritable)o).get()); + return o == null ? null : Integer.valueOf(((IntWritable) o).get()); } - + @Override public Object create(int value) { return new IntWritable(value); @@ -53,7 +52,7 @@ @Override public Object set(Object o, int value) { - ((IntWritable)o).set(value); + ((IntWritable) o).set(value); return o; } } Index: serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/AbstractPrimitiveWritableObjectInspector.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/AbstractPrimitiveWritableObjectInspector.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/AbstractPrimitiveWritableObjectInspector.java (working copy) @@ -20,14 +20,16 @@ import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils.PrimitiveTypeEntry; /** - * An AbstractWritablePrimitiveObjectInspector for a Writable object. + * An AbstractWritablePrimitiveObjectInspector for a Writable object. */ -public abstract class AbstractPrimitiveWritableObjectInspector extends AbstractPrimitiveObjectInspector { +public abstract class AbstractPrimitiveWritableObjectInspector extends + AbstractPrimitiveObjectInspector { - protected AbstractPrimitiveWritableObjectInspector(PrimitiveTypeEntry typeEntry) { - super(typeEntry); + protected AbstractPrimitiveWritableObjectInspector( + PrimitiveTypeEntry typeEntry) { + super(typeEntry); } - + @Override public Object getPrimitiveWritableObject(Object o) { return o; @@ -37,5 +39,5 @@ public boolean preferWritable() { return true; } - + } Index: serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/SettableByteObjectInspector.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/SettableByteObjectInspector.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/SettableByteObjectInspector.java (working copy) @@ -17,7 +17,6 @@ */ package org.apache.hadoop.hive.serde2.objectinspector.primitive; - /** * A SettableByteObjectInspector can set a byte value to an object. */ @@ -26,11 +25,11 @@ /** * Set the object with the value. Return the object that has the new value. * - * In most cases the returned value should be the same as o, but in case - * o is unmodifiable, this will return a new object with new value. + * In most cases the returned value should be the same as o, but in case o is + * unmodifiable, this will return a new object with new value. */ public Object set(Object o, byte value); - + /** * Create an object with the value. */ Index: serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/SettableFloatObjectInspector.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/SettableFloatObjectInspector.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/SettableFloatObjectInspector.java (working copy) @@ -17,7 +17,6 @@ */ package org.apache.hadoop.hive.serde2.objectinspector.primitive; - /** * A SettableFloatObjectInspector can set a float value to an object. */ @@ -26,11 +25,11 @@ /** * Set the object with the value. Return the object that has the new value. * - * In most cases the returned value should be the same as o, but in case - * o is unmodifiable, this will return a new object with new value. + * In most cases the returned value should be the same as o, but in case o is + * unmodifiable, this will return a new object with new value. */ public Object set(Object o, float value); - + /** * Create an object with the value. */ Index: serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/SettableStringObjectInspector.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/SettableStringObjectInspector.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/SettableStringObjectInspector.java (working copy) @@ -19,7 +19,6 @@ import org.apache.hadoop.io.Text; - /** * A SettableStringObjectInspector can set a string value to an object. */ @@ -28,19 +27,19 @@ /** * Set the object with the value. Return the object that has the new value. * - * In most cases the returned value should be the same as o, but in case - * o is unmodifiable, this will return a new object with new value. + * In most cases the returned value should be the same as o, but in case o is + * unmodifiable, this will return a new object with new value. */ public Object set(Object o, Text value); - + /** * Set the object with the value. Return the object that has the new value. * - * In most cases the returned value should be the same as o, but in case - * o is unmodifiable, this will return a new object with new value. + * In most cases the returned value should be the same as o, but in case o is + * unmodifiable, this will return a new object with new value. */ public Object set(Object o, String value); - + /** * Create an object with the value. */ Index: serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableVoidObjectInspector.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableVoidObjectInspector.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableVoidObjectInspector.java (working copy) @@ -17,17 +17,16 @@ */ package org.apache.hadoop.hive.serde2.objectinspector.primitive; - /** * A WritableVoidObjectInspector inspects a NullWritable Object. */ -public class WritableVoidObjectInspector extends AbstractPrimitiveWritableObjectInspector -implements VoidObjectInspector{ +public class WritableVoidObjectInspector extends + AbstractPrimitiveWritableObjectInspector implements VoidObjectInspector { WritableVoidObjectInspector() { super(PrimitiveObjectInspectorUtils.voidTypeEntry); } - + @Override public Object copyObject(Object o) { return o; Index: serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/ShortObjectInspector.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/ShortObjectInspector.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/ShortObjectInspector.java (working copy) @@ -19,12 +19,11 @@ import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; - /** * A ShortObjectInspector inspects an Object representing a Short. */ public interface ShortObjectInspector extends PrimitiveObjectInspector { - + /** * Get the short data. */ Index: serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/LongObjectInspector.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/LongObjectInspector.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/LongObjectInspector.java (working copy) @@ -19,12 +19,11 @@ import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; - /** * A LongObjectInspector inspects an Object representing a Long. */ public interface LongObjectInspector extends PrimitiveObjectInspector { - + /** * Get the long data. */ Index: serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/JavaLongObjectInspector.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/JavaLongObjectInspector.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/JavaLongObjectInspector.java (working copy) @@ -19,25 +19,24 @@ import org.apache.hadoop.io.LongWritable; - /** * A JavaLongObjectInspector inspects a Java Long Object. */ -public class JavaLongObjectInspector extends AbstractPrimitiveJavaObjectInspector -implements SettableLongObjectInspector{ +public class JavaLongObjectInspector extends + AbstractPrimitiveJavaObjectInspector implements SettableLongObjectInspector { JavaLongObjectInspector() { super(PrimitiveObjectInspectorUtils.longTypeEntry); } - + @Override public Object getPrimitiveWritableObject(Object o) { - return o == null ? null : new LongWritable(((Long)o).longValue()); + return o == null ? null : new LongWritable(((Long) o).longValue()); } @Override public long get(Object o) { - return ((Long)o).longValue(); + return ((Long) o).longValue(); } @Override Index: serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/JavaShortObjectInspector.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/JavaShortObjectInspector.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/JavaShortObjectInspector.java (working copy) @@ -19,25 +19,25 @@ import org.apache.hadoop.hive.serde2.io.ShortWritable; - /** * A JavaShortObjectInspector inspects a Java Short Object. */ -public class JavaShortObjectInspector extends AbstractPrimitiveJavaObjectInspector -implements SettableShortObjectInspector{ +public class JavaShortObjectInspector extends + AbstractPrimitiveJavaObjectInspector implements + SettableShortObjectInspector { JavaShortObjectInspector() { super(PrimitiveObjectInspectorUtils.shortTypeEntry); } - + @Override public Object getPrimitiveWritableObject(Object o) { - return o == null ? null : new ShortWritable(((Short)o).shortValue()); + return o == null ? null : new ShortWritable(((Short) o).shortValue()); } @Override public short get(Object o) { - return ((Short)o).shortValue(); + return ((Short) o).shortValue(); } @Override Index: serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/ByteObjectInspector.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/ByteObjectInspector.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/ByteObjectInspector.java (working copy) @@ -19,15 +19,14 @@ import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; - /** * A ByteObjectInspector inspects an Object representing a Byte object. */ public interface ByteObjectInspector extends PrimitiveObjectInspector { - + /** * Get the byte data. */ byte get(Object o); - + } Index: serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableDoubleObjectInspector.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableDoubleObjectInspector.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableDoubleObjectInspector.java (working copy) @@ -17,33 +17,32 @@ */ package org.apache.hadoop.hive.serde2.objectinspector.primitive; -import org.apache.hadoop.hive.serde2.io.ByteWritable; import org.apache.hadoop.hive.serde2.io.DoubleWritable; - /** * A WritableDoubleObjectInspector inspects a DoubleWritable Object. */ -public class WritableDoubleObjectInspector extends AbstractPrimitiveWritableObjectInspector -implements SettableDoubleObjectInspector{ +public class WritableDoubleObjectInspector extends + AbstractPrimitiveWritableObjectInspector implements + SettableDoubleObjectInspector { WritableDoubleObjectInspector() { super(PrimitiveObjectInspectorUtils.doubleTypeEntry); } - + @Override public double get(Object o) { - return ((DoubleWritable)o).get(); + return ((DoubleWritable) o).get(); } @Override public Object copyObject(Object o) { - return o == null ? null : new DoubleWritable(((DoubleWritable)o).get()); + return o == null ? null : new DoubleWritable(((DoubleWritable) o).get()); } @Override public Object getPrimitiveJavaObject(Object o) { - return o == null ? null : Double.valueOf(((DoubleWritable)o).get()); + return o == null ? null : Double.valueOf(((DoubleWritable) o).get()); } @Override @@ -53,8 +52,8 @@ @Override public Object set(Object o, double value) { - ((DoubleWritable)o).set(value); + ((DoubleWritable) o).set(value); return o; } - + } Index: serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/JavaByteObjectInspector.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/JavaByteObjectInspector.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/JavaByteObjectInspector.java (working copy) @@ -19,25 +19,24 @@ import org.apache.hadoop.hive.serde2.io.ByteWritable; - /** * A JavaByteObjectInspector inspects a Java Byte Object. */ -public class JavaByteObjectInspector extends AbstractPrimitiveJavaObjectInspector -implements SettableByteObjectInspector{ +public class JavaByteObjectInspector extends + AbstractPrimitiveJavaObjectInspector implements SettableByteObjectInspector { JavaByteObjectInspector() { super(PrimitiveObjectInspectorUtils.byteTypeEntry); } - + @Override public Object getPrimitiveWritableObject(Object o) { - return o == null ? null : new ByteWritable(((Byte)o).byteValue()); + return o == null ? null : new ByteWritable(((Byte) o).byteValue()); } @Override public byte get(Object o) { - return ((Byte)o).byteValue(); + return ((Byte) o).byteValue(); } @Override Index: serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/FloatObjectInspector.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/FloatObjectInspector.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/FloatObjectInspector.java (working copy) @@ -19,12 +19,11 @@ import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; - /** * A FloatObjectInspector inspects an Object representing a Float. */ public interface FloatObjectInspector extends PrimitiveObjectInspector { - + /** * Get the float data. */ Index: serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/StringObjectInspector.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/StringObjectInspector.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/StringObjectInspector.java (working copy) @@ -20,17 +20,16 @@ import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; import org.apache.hadoop.io.Text; - /** * A StringObjectInspector inspects an Object representing a String. */ public interface StringObjectInspector extends PrimitiveObjectInspector { - + /** * Get the Text representation of the data. */ Text getPrimitiveWritableObject(Object o); - + /** * Get the String representation of the data. */ Index: serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/JavaFloatObjectInspector.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/JavaFloatObjectInspector.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/JavaFloatObjectInspector.java (working copy) @@ -19,25 +19,25 @@ import org.apache.hadoop.io.FloatWritable; - /** * A JavaFloatObjectInspector inspects a Java Float Object. */ -public class JavaFloatObjectInspector extends AbstractPrimitiveJavaObjectInspector -implements SettableFloatObjectInspector{ +public class JavaFloatObjectInspector extends + AbstractPrimitiveJavaObjectInspector implements + SettableFloatObjectInspector { JavaFloatObjectInspector() { super(PrimitiveObjectInspectorUtils.floatTypeEntry); } - + @Override public Object getPrimitiveWritableObject(Object o) { - return o == null ? null : new FloatWritable(((Float)o).floatValue()); + return o == null ? null : new FloatWritable(((Float) o).floatValue()); } @Override public float get(Object o) { - return ((Float)o).floatValue(); + return ((Float) o).floatValue(); } @Override Index: serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/JavaStringObjectInspector.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/JavaStringObjectInspector.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/JavaStringObjectInspector.java (working copy) @@ -19,25 +19,25 @@ import org.apache.hadoop.io.Text; - /** * A JavaStringObjectInspector inspects a Java String Object. */ -public class JavaStringObjectInspector extends AbstractPrimitiveJavaObjectInspector -implements SettableStringObjectInspector{ +public class JavaStringObjectInspector extends + AbstractPrimitiveJavaObjectInspector implements + SettableStringObjectInspector { JavaStringObjectInspector() { super(PrimitiveObjectInspectorUtils.stringTypeEntry); } - + @Override public Text getPrimitiveWritableObject(Object o) { - return o == null ? null : new Text(((String)o)); + return o == null ? null : new Text(((String) o)); } @Override public String getPrimitiveJavaObject(Object o) { - return (String)o; + return (String) o; } @Override Index: serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableBooleanObjectInspector.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableBooleanObjectInspector.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableBooleanObjectInspector.java (working copy) @@ -19,30 +19,30 @@ import org.apache.hadoop.io.BooleanWritable; - /** * A WritableBooleanObjectInspector inspects a BooleanWritable Object. */ -public class WritableBooleanObjectInspector extends AbstractPrimitiveWritableObjectInspector -implements SettableBooleanObjectInspector{ +public class WritableBooleanObjectInspector extends + AbstractPrimitiveWritableObjectInspector implements + SettableBooleanObjectInspector { WritableBooleanObjectInspector() { super(PrimitiveObjectInspectorUtils.booleanTypeEntry); } - + @Override public boolean get(Object o) { - return ((BooleanWritable)o).get(); + return ((BooleanWritable) o).get(); } @Override public Object copyObject(Object o) { - return o == null ? null : new BooleanWritable(((BooleanWritable)o).get()); + return o == null ? null : new BooleanWritable(((BooleanWritable) o).get()); } @Override public Object getPrimitiveJavaObject(Object o) { - return o == null ? null : Boolean.valueOf(((BooleanWritable)o).get()); + return o == null ? null : Boolean.valueOf(((BooleanWritable) o).get()); } @Override @@ -52,7 +52,7 @@ @Override public Object set(Object o, boolean value) { - ((BooleanWritable)o).set(value); + ((BooleanWritable) o).set(value); return o; } } Index: serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/SettableIntObjectInspector.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/SettableIntObjectInspector.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/SettableIntObjectInspector.java (working copy) @@ -17,7 +17,6 @@ */ package org.apache.hadoop.hive.serde2.objectinspector.primitive; - /** * A SettableIntObjectInspector can set an int value to an object. */ @@ -26,11 +25,11 @@ /** * Set the object with the value. Return the object that has the new value. * - * In most cases the returned value should be the same as o, but in case - * o is unmodifiable, this will return a new object with new value. + * In most cases the returned value should be the same as o, but in case o is + * unmodifiable, this will return a new object with new value. */ public Object set(Object o, int value); - + /** * Create an object with the value. */ Index: serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableShortObjectInspector.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableShortObjectInspector.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableShortObjectInspector.java (working copy) @@ -22,28 +22,29 @@ /** * A WritableShortObjectInspector inspects a ShortWritable Object. */ -public class WritableShortObjectInspector extends AbstractPrimitiveWritableObjectInspector -implements SettableShortObjectInspector{ +public class WritableShortObjectInspector extends + AbstractPrimitiveWritableObjectInspector implements + SettableShortObjectInspector { WritableShortObjectInspector() { super(PrimitiveObjectInspectorUtils.shortTypeEntry); } - + @Override public short get(Object o) { - return ((ShortWritable)o).get(); + return ((ShortWritable) o).get(); } @Override public Object copyObject(Object o) { - return o == null ? null : new ShortWritable(((ShortWritable)o).get()); + return o == null ? null : new ShortWritable(((ShortWritable) o).get()); } @Override public Object getPrimitiveJavaObject(Object o) { - return o == null ? null : Short.valueOf(((ShortWritable)o).get()); + return o == null ? null : Short.valueOf(((ShortWritable) o).get()); } - + @Override public Object create(short value) { return new ShortWritable(value); @@ -51,8 +52,8 @@ @Override public Object set(Object o, short value) { - ((ShortWritable)o).set(value); + ((ShortWritable) o).set(value); return o; } - + } Index: serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableLongObjectInspector.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableLongObjectInspector.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableLongObjectInspector.java (working copy) @@ -17,35 +17,34 @@ */ package org.apache.hadoop.hive.serde2.objectinspector.primitive; -import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.io.LongWritable; - /** * A WritableLongObjectInspector inspects a LongWritable Object. */ -public class WritableLongObjectInspector extends AbstractPrimitiveWritableObjectInspector -implements SettableLongObjectInspector{ +public class WritableLongObjectInspector extends + AbstractPrimitiveWritableObjectInspector implements + SettableLongObjectInspector { WritableLongObjectInspector() { super(PrimitiveObjectInspectorUtils.longTypeEntry); } - + @Override public long get(Object o) { - return ((LongWritable)o).get(); + return ((LongWritable) o).get(); } @Override public Object copyObject(Object o) { - return o == null ? null : new LongWritable(((LongWritable)o).get()); + return o == null ? null : new LongWritable(((LongWritable) o).get()); } @Override public Object getPrimitiveJavaObject(Object o) { - return o == null ? null : Long.valueOf(((LongWritable)o).get()); + return o == null ? null : Long.valueOf(((LongWritable) o).get()); } - + @Override public Object create(long value) { return new LongWritable(value); @@ -53,8 +52,8 @@ @Override public Object set(Object o, long value) { - ((LongWritable)o).set(value); + ((LongWritable) o).set(value); return o; } - + } Index: serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorUtils.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorUtils.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorUtils.java (working copy) @@ -19,7 +19,6 @@ package org.apache.hadoop.hive.serde2.objectinspector.primitive; import java.util.HashMap; -import java.util.List; import java.util.Map; import org.apache.commons.logging.Log; @@ -31,7 +30,6 @@ import org.apache.hadoop.hive.serde2.lazy.LazyInteger; import org.apache.hadoop.hive.serde2.lazy.LazyLong; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils.ObjectInspectorCopyOption; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; import org.apache.hadoop.io.BooleanWritable; import org.apache.hadoop.io.FloatWritable; @@ -42,58 +40,64 @@ import org.apache.hadoop.io.Writable; /** - * ObjectInspectorFactory is the primary way to create new ObjectInspector instances. + * ObjectInspectorFactory is the primary way to create new ObjectInspector + * instances. * - * SerDe classes should call the static functions in this library to create an ObjectInspector - * to return to the caller of SerDe2.getObjectInspector(). + * SerDe classes should call the static functions in this library to create an + * ObjectInspector to return to the caller of SerDe2.getObjectInspector(). */ public class PrimitiveObjectInspectorUtils { - private static Log LOG = LogFactory.getLog(PrimitiveObjectInspectorUtils.class.getName()); - + private static Log LOG = LogFactory + .getLog(PrimitiveObjectInspectorUtils.class.getName()); + /** * TypeEntry stores information about a Hive Primitive TypeInfo. */ public static class PrimitiveTypeEntry { - + /** * The category of the PrimitiveType. */ public PrimitiveObjectInspector.PrimitiveCategory primitiveCategory; - + /** - * primitiveJavaType refers to java types like int, double, etc. + * primitiveJavaType refers to java types like int, double, etc. */ public Class primitiveJavaType; /** - * primitiveJavaClass refers to java classes like Integer, Double, String etc. + * primitiveJavaClass refers to java classes like Integer, Double, String + * etc. */ public Class primitiveJavaClass; /** - * writableClass refers to hadoop Writable classes like IntWritable, DoubleWritable, Text etc. + * writableClass refers to hadoop Writable classes like IntWritable, + * DoubleWritable, Text etc. */ public Class primitiveWritableClass; /** * typeName is the name of the type as in DDL. */ public String typeName; - - PrimitiveTypeEntry(PrimitiveObjectInspector.PrimitiveCategory primitiveCategory, - String typeName, Class primitiveType, Class javaClass, Class hiveClass) { + + PrimitiveTypeEntry( + PrimitiveObjectInspector.PrimitiveCategory primitiveCategory, + String typeName, Class primitiveType, Class javaClass, + Class hiveClass) { this.primitiveCategory = primitiveCategory; - this.primitiveJavaType = primitiveType; - this.primitiveJavaClass = javaClass; - this.primitiveWritableClass = hiveClass; + primitiveJavaType = primitiveType; + primitiveJavaClass = javaClass; + primitiveWritableClass = hiveClass; this.typeName = typeName; } } - + static final Map primitiveCategoryToTypeEntry = new HashMap(); static final Map, PrimitiveTypeEntry> primitiveJavaTypeToTypeEntry = new HashMap, PrimitiveTypeEntry>(); static final Map, PrimitiveTypeEntry> primitiveJavaClassToTypeEntry = new HashMap, PrimitiveTypeEntry>(); static final Map, PrimitiveTypeEntry> primitiveWritableClassToTypeEntry = new HashMap, PrimitiveTypeEntry>(); static final Map typeNameToTypeEntry = new HashMap(); - + static void registerType(PrimitiveTypeEntry t) { if (t.primitiveCategory != PrimitiveCategory.UNKNOWN) { primitiveCategoryToTypeEntry.put(t.primitiveCategory, t); @@ -111,22 +115,41 @@ typeNameToTypeEntry.put(t.typeName, t); } } - - public static final PrimitiveTypeEntry stringTypeEntry = new PrimitiveTypeEntry(PrimitiveCategory.STRING, Constants.STRING_TYPE_NAME, null, String.class, Text.class); - public static final PrimitiveTypeEntry booleanTypeEntry = new PrimitiveTypeEntry(PrimitiveCategory.BOOLEAN, Constants.BOOLEAN_TYPE_NAME, Boolean.TYPE, Boolean.class, BooleanWritable.class); - public static final PrimitiveTypeEntry intTypeEntry = new PrimitiveTypeEntry(PrimitiveCategory.INT, Constants.INT_TYPE_NAME, Integer.TYPE, Integer.class, IntWritable.class); - public static final PrimitiveTypeEntry longTypeEntry = new PrimitiveTypeEntry(PrimitiveCategory.LONG, Constants.BIGINT_TYPE_NAME, Long.TYPE, Long.class, LongWritable.class); - public static final PrimitiveTypeEntry floatTypeEntry = new PrimitiveTypeEntry(PrimitiveCategory.FLOAT, Constants.FLOAT_TYPE_NAME, Float.TYPE, Float.class, FloatWritable.class); - public static final PrimitiveTypeEntry voidTypeEntry = new PrimitiveTypeEntry(PrimitiveCategory.VOID, Constants.VOID_TYPE_NAME, Void.TYPE, Void.class, NullWritable.class); + public static final PrimitiveTypeEntry stringTypeEntry = new PrimitiveTypeEntry( + PrimitiveCategory.STRING, Constants.STRING_TYPE_NAME, null, String.class, + Text.class); + public static final PrimitiveTypeEntry booleanTypeEntry = new PrimitiveTypeEntry( + PrimitiveCategory.BOOLEAN, Constants.BOOLEAN_TYPE_NAME, Boolean.TYPE, + Boolean.class, BooleanWritable.class); + public static final PrimitiveTypeEntry intTypeEntry = new PrimitiveTypeEntry( + PrimitiveCategory.INT, Constants.INT_TYPE_NAME, Integer.TYPE, + Integer.class, IntWritable.class); + public static final PrimitiveTypeEntry longTypeEntry = new PrimitiveTypeEntry( + PrimitiveCategory.LONG, Constants.BIGINT_TYPE_NAME, Long.TYPE, + Long.class, LongWritable.class); + public static final PrimitiveTypeEntry floatTypeEntry = new PrimitiveTypeEntry( + PrimitiveCategory.FLOAT, Constants.FLOAT_TYPE_NAME, Float.TYPE, + Float.class, FloatWritable.class); + public static final PrimitiveTypeEntry voidTypeEntry = new PrimitiveTypeEntry( + PrimitiveCategory.VOID, Constants.VOID_TYPE_NAME, Void.TYPE, Void.class, + NullWritable.class); + // No corresponding Writable classes for the following 3 in hadoop 0.17.0 - public static final PrimitiveTypeEntry doubleTypeEntry = new PrimitiveTypeEntry(PrimitiveCategory.DOUBLE, Constants.DOUBLE_TYPE_NAME, Double.TYPE, Double.class, DoubleWritable.class); - public static final PrimitiveTypeEntry byteTypeEntry = new PrimitiveTypeEntry(PrimitiveCategory.BYTE, Constants.TINYINT_TYPE_NAME, Byte.TYPE, Byte.class, ByteWritable.class); - public static final PrimitiveTypeEntry shortTypeEntry = new PrimitiveTypeEntry(PrimitiveCategory.SHORT, Constants.SMALLINT_TYPE_NAME, Short.TYPE, Short.class, ShortWritable.class); + public static final PrimitiveTypeEntry doubleTypeEntry = new PrimitiveTypeEntry( + PrimitiveCategory.DOUBLE, Constants.DOUBLE_TYPE_NAME, Double.TYPE, + Double.class, DoubleWritable.class); + public static final PrimitiveTypeEntry byteTypeEntry = new PrimitiveTypeEntry( + PrimitiveCategory.BYTE, Constants.TINYINT_TYPE_NAME, Byte.TYPE, + Byte.class, ByteWritable.class); + public static final PrimitiveTypeEntry shortTypeEntry = new PrimitiveTypeEntry( + PrimitiveCategory.SHORT, Constants.SMALLINT_TYPE_NAME, Short.TYPE, + Short.class, ShortWritable.class); // The following is a complex type for special handling - public static final PrimitiveTypeEntry unknownTypeEntry = new PrimitiveTypeEntry(PrimitiveCategory.UNKNOWN, "unknown", null, Object.class, null); - + public static final PrimitiveTypeEntry unknownTypeEntry = new PrimitiveTypeEntry( + PrimitiveCategory.UNKNOWN, "unknown", null, Object.class, null); + static { registerType(stringTypeEntry); registerType(booleanTypeEntry); @@ -141,7 +164,8 @@ } /** - * Return Whether the class is a Java Primitive type or a Java Primitive class. + * Return Whether the class is a Java Primitive type or a Java Primitive + * class. */ public static Class primitiveJavaTypeToClass(Class clazz) { PrimitiveTypeEntry t = primitiveJavaTypeToTypeEntry.get(clazz); @@ -149,36 +173,36 @@ } /** - * Whether the class is a Java Primitive type or a Java Primitive class. + * Whether the class is a Java Primitive type or a Java Primitive class. */ public static boolean isPrimitiveJava(Class clazz) { return primitiveJavaTypeToTypeEntry.get(clazz) != null - || primitiveJavaClassToTypeEntry.get(clazz) != null; + || primitiveJavaClassToTypeEntry.get(clazz) != null; } - + /** - * Whether the class is a Java Primitive type. + * Whether the class is a Java Primitive type. */ public static boolean isPrimitiveJavaType(Class clazz) { return primitiveJavaTypeToTypeEntry.get(clazz) != null; } /** - * Whether the class is a Java Primitive class. + * Whether the class is a Java Primitive class. */ public static boolean isPrimitiveJavaClass(Class clazz) { return primitiveJavaClassToTypeEntry.get(clazz) != null; } /** - * Whether the class is a Hive Primitive Writable class. + * Whether the class is a Hive Primitive Writable class. */ public static boolean isPrimitiveWritableClass(Class clazz) { return primitiveWritableClassToTypeEntry.get(clazz) != null; } - + /** - * Get the typeName from a Java Primitive Type or Java PrimitiveClass. + * Get the typeName from a Java Primitive Type or Java PrimitiveClass. */ public static String getTypeNameFromPrimitiveJava(Class clazz) { PrimitiveTypeEntry t = primitiveJavaTypeToTypeEntry.get(clazz); @@ -187,9 +211,9 @@ } return t == null ? null : t.typeName; } - + /** - * Get the typeName from a Primitive Writable Class. + * Get the typeName from a Primitive Writable Class. */ public static String getTypeNameFromPrimitiveWritable(Class clazz) { PrimitiveTypeEntry t = primitiveWritableClassToTypeEntry.get(clazz); @@ -197,14 +221,15 @@ } /** - * Get the typeName from a Java Primitive Type or Java PrimitiveClass. + * Get the typeName from a Java Primitive Type or Java PrimitiveClass. */ - public static PrimitiveTypeEntry getTypeEntryFromPrimitiveCategory(PrimitiveCategory category) { + public static PrimitiveTypeEntry getTypeEntryFromPrimitiveCategory( + PrimitiveCategory category) { return primitiveCategoryToTypeEntry.get(category); } - + /** - * Get the TypeEntry for a Java Primitive Type or Java PrimitiveClass. + * Get the TypeEntry for a Java Primitive Type or Java PrimitiveClass. */ public static PrimitiveTypeEntry getTypeEntryFromPrimitiveJava(Class clazz) { PrimitiveTypeEntry t = primitiveJavaTypeToTypeEntry.get(clazz); @@ -213,449 +238,473 @@ } return t; } - + /** - * Get the TypeEntry for a Java Primitive Type or Java PrimitiveClass. + * Get the TypeEntry for a Java Primitive Type or Java PrimitiveClass. */ - public static PrimitiveTypeEntry getTypeEntryFromPrimitiveJavaType(Class clazz) { + public static PrimitiveTypeEntry getTypeEntryFromPrimitiveJavaType( + Class clazz) { return primitiveJavaTypeToTypeEntry.get(clazz); } - + /** - * Get the TypeEntry for a Java Primitive Type or Java PrimitiveClass. + * Get the TypeEntry for a Java Primitive Type or Java PrimitiveClass. */ - public static PrimitiveTypeEntry getTypeEntryFromPrimitiveJavaClass(Class clazz) { + public static PrimitiveTypeEntry getTypeEntryFromPrimitiveJavaClass( + Class clazz) { return primitiveJavaClassToTypeEntry.get(clazz); } - + /** - * Get the TypeEntry for a Primitive Writable Class. + * Get the TypeEntry for a Primitive Writable Class. */ - public static PrimitiveTypeEntry getTypeEntryFromPrimitiveWritableClass(Class clazz) { + public static PrimitiveTypeEntry getTypeEntryFromPrimitiveWritableClass( + Class clazz) { return primitiveWritableClassToTypeEntry.get(clazz); } - + /** - * Get the TypeEntry for a Primitive Writable Class. + * Get the TypeEntry for a Primitive Writable Class. */ public static PrimitiveTypeEntry getTypeEntryFromTypeName(String typeName) { return typeNameToTypeEntry.get(typeName); } - + /** - * Compare 2 primitive objects. Conversion not allowed. - * Note that NULL does not equal to NULL according to SQL standard. + * Compare 2 primitive objects. Conversion not allowed. Note that NULL does + * not equal to NULL according to SQL standard. */ - public static boolean comparePrimitiveObjects(Object o1, PrimitiveObjectInspector oi1, - Object o2, PrimitiveObjectInspector oi2) { - if (o1 == null || o2 == null) return false; - + public static boolean comparePrimitiveObjects(Object o1, + PrimitiveObjectInspector oi1, Object o2, PrimitiveObjectInspector oi2) { + if (o1 == null || o2 == null) { + return false; + } + if (oi1.getPrimitiveCategory() != oi2.getPrimitiveCategory()) { return false; } switch (oi1.getPrimitiveCategory()) { - case BOOLEAN: { - return ((BooleanObjectInspector)oi1).get(o1) == ((BooleanObjectInspector)oi2).get(o2); - } - case BYTE: { - return ((ByteObjectInspector)oi1).get(o1) == ((ByteObjectInspector)oi2).get(o2); - } - case SHORT: { - return ((ShortObjectInspector)oi1).get(o1) == ((ShortObjectInspector)oi2).get(o2); - } - case INT: { - return ((IntObjectInspector)oi1).get(o1) == ((IntObjectInspector)oi2).get(o2); - } - case LONG: { - return ((LongObjectInspector)oi1).get(o1) == ((LongObjectInspector)oi2).get(o2); - } - case FLOAT: { - return ((FloatObjectInspector)oi1).get(o1) == ((FloatObjectInspector)oi2).get(o2); - } - case DOUBLE: { - return ((DoubleObjectInspector)oi1).get(o1) == ((DoubleObjectInspector)oi2).get(o2); - } - case STRING: { - Writable t1 = ((StringObjectInspector)oi1).getPrimitiveWritableObject(o1); - Writable t2 = ((StringObjectInspector)oi2).getPrimitiveWritableObject(o2); - return t1.equals(t2); - } - default: - return false; + case BOOLEAN: { + return ((BooleanObjectInspector) oi1).get(o1) == ((BooleanObjectInspector) oi2) + .get(o2); } + case BYTE: { + return ((ByteObjectInspector) oi1).get(o1) == ((ByteObjectInspector) oi2) + .get(o2); + } + case SHORT: { + return ((ShortObjectInspector) oi1).get(o1) == ((ShortObjectInspector) oi2) + .get(o2); + } + case INT: { + return ((IntObjectInspector) oi1).get(o1) == ((IntObjectInspector) oi2) + .get(o2); + } + case LONG: { + return ((LongObjectInspector) oi1).get(o1) == ((LongObjectInspector) oi2) + .get(o2); + } + case FLOAT: { + return ((FloatObjectInspector) oi1).get(o1) == ((FloatObjectInspector) oi2) + .get(o2); + } + case DOUBLE: { + return ((DoubleObjectInspector) oi1).get(o1) == ((DoubleObjectInspector) oi2) + .get(o2); + } + case STRING: { + Writable t1 = ((StringObjectInspector) oi1) + .getPrimitiveWritableObject(o1); + Writable t2 = ((StringObjectInspector) oi2) + .getPrimitiveWritableObject(o2); + return t1.equals(t2); + } + default: + return false; + } } - - /** Convert a primitive object to double. + /** + * Convert a primitive object to double. */ - public static double convertPrimitiveToDouble(Object o, PrimitiveObjectInspector oi) - throws NumberFormatException { + public static double convertPrimitiveToDouble(Object o, + PrimitiveObjectInspector oi) throws NumberFormatException { switch (oi.getPrimitiveCategory()) { - case BOOLEAN: { - return ((BooleanObjectInspector)oi).get(o) ? 1 : 0; - } - case BYTE: { - return ((ByteObjectInspector)oi).get(o); - } - case SHORT: { - return ((ShortObjectInspector)oi).get(o); - } - case INT: { - return ((IntObjectInspector)oi).get(o); - } - case LONG: { - return ((LongObjectInspector)oi).get(o); - } - case FLOAT: { - return ((FloatObjectInspector)oi).get(o); - } - case DOUBLE: { - return ((DoubleObjectInspector)oi).get(o); - } - case STRING: { - return Double.valueOf(((StringObjectInspector)oi).getPrimitiveJavaObject(o)); - } - default: - throw new NumberFormatException(); + case BOOLEAN: { + return ((BooleanObjectInspector) oi).get(o) ? 1 : 0; } + case BYTE: { + return ((ByteObjectInspector) oi).get(o); + } + case SHORT: { + return ((ShortObjectInspector) oi).get(o); + } + case INT: { + return ((IntObjectInspector) oi).get(o); + } + case LONG: { + return ((LongObjectInspector) oi).get(o); + } + case FLOAT: { + return ((FloatObjectInspector) oi).get(o); + } + case DOUBLE: { + return ((DoubleObjectInspector) oi).get(o); + } + case STRING: { + return Double.valueOf(((StringObjectInspector) oi) + .getPrimitiveJavaObject(o)); + } + default: + throw new NumberFormatException(); + } } - + /** - * Compare 2 Primitive Objects with their Object Inspector, conversions allowed. - * Note that NULL does not equal to NULL according to SQL standard. + * Compare 2 Primitive Objects with their Object Inspector, conversions + * allowed. Note that NULL does not equal to NULL according to SQL standard. */ - public static boolean comparePrimitiveObjectsWithConversion(Object o1, PrimitiveObjectInspector oi1, - Object o2, PrimitiveObjectInspector oi2) { - if (o1 == null || o2 == null) return false; - + public static boolean comparePrimitiveObjectsWithConversion(Object o1, + PrimitiveObjectInspector oi1, Object o2, PrimitiveObjectInspector oi2) { + if (o1 == null || o2 == null) { + return false; + } + if (oi1.getPrimitiveCategory() == oi2.getPrimitiveCategory()) { return comparePrimitiveObjects(o1, oi1, o2, oi2); } - + // If not equal, convert all to double and compare try { - return convertPrimitiveToDouble(o1, oi1) == convertPrimitiveToDouble(o2, oi2); + return convertPrimitiveToDouble(o1, oi1) == convertPrimitiveToDouble(o2, + oi2); } catch (NumberFormatException e) { return false; } } - + /** - * Get the boolean value out of a primitive object. - * Note that NullPointerException will be thrown if o is null. - * Note that NumberFormatException will be thrown if o is not a valid number. + * Get the boolean value out of a primitive object. Note that + * NullPointerException will be thrown if o is null. Note that + * NumberFormatException will be thrown if o is not a valid number. */ - public static boolean getBoolean(Object o, PrimitiveObjectInspector oi) throws NumberFormatException { + public static boolean getBoolean(Object o, PrimitiveObjectInspector oi) + throws NumberFormatException { boolean result = false; switch (oi.getPrimitiveCategory()) { - case VOID: { - result = false; - break; + case VOID: { + result = false; + break; + } + case BOOLEAN: { + result = ((BooleanObjectInspector) oi).get(o); + break; + } + case BYTE: { + result = ((ByteObjectInspector) oi).get(o) != 0; + break; + } + case SHORT: { + result = ((ShortObjectInspector) oi).get(o) != 0; + break; + } + case INT: { + result = ((IntObjectInspector) oi).get(o) != 0; + break; + } + case LONG: { + result = (int) ((LongObjectInspector) oi).get(o) != 0; + break; + } + case FLOAT: { + result = (int) ((FloatObjectInspector) oi).get(o) != 0; + break; + } + case DOUBLE: { + result = (int) ((DoubleObjectInspector) oi).get(o) != 0; + break; + } + case STRING: { + StringObjectInspector soi = (StringObjectInspector) oi; + if (soi.preferWritable()) { + Text t = soi.getPrimitiveWritableObject(o); + result = t.getLength() != 0; + } else { + String s = soi.getPrimitiveJavaObject(o); + result = s.length() != 0; } - case BOOLEAN: { - result = ((BooleanObjectInspector)oi).get(o); - break; - } - case BYTE: { - result = ((ByteObjectInspector)oi).get(o) != 0; - break; - } - case SHORT: { - result = ((ShortObjectInspector)oi).get(o) != 0; - break; - } - case INT: { - result = ((IntObjectInspector)oi).get(o) != 0; - break; - } - case LONG: { - result = (int)((LongObjectInspector)oi).get(o) != 0; - break; - } - case FLOAT: { - result = (int)((FloatObjectInspector)oi).get(o) != 0; - break; - } - case DOUBLE: { - result = (int)((DoubleObjectInspector)oi).get(o) != 0; - break; - } - case STRING: { - StringObjectInspector soi = (StringObjectInspector)oi; - if (soi.preferWritable()) { - Text t = soi.getPrimitiveWritableObject(o); - result = t.getLength() != 0; - } else { - String s = soi.getPrimitiveJavaObject(o); - result = s.length() != 0; - } - break; - } - default: { - throw new RuntimeException("Hive 2 Internal error: unknown type: " - + oi.getTypeName()); - } + break; } + default: { + throw new RuntimeException("Hive 2 Internal error: unknown type: " + + oi.getTypeName()); + } + } return result; } - + /** - * Get the byte value out of a primitive object. - * Note that NullPointerException will be thrown if o is null. - * Note that NumberFormatException will be thrown if o is not a valid number. + * Get the byte value out of a primitive object. Note that + * NullPointerException will be thrown if o is null. Note that + * NumberFormatException will be thrown if o is not a valid number. */ - public static byte getByte(Object o, PrimitiveObjectInspector oi) throws NumberFormatException { - return (byte)getInt(o, oi); + public static byte getByte(Object o, PrimitiveObjectInspector oi) + throws NumberFormatException { + return (byte) getInt(o, oi); } - + /** - * Get the short value out of a primitive object. - * Note that NullPointerException will be thrown if o is null. - * Note that NumberFormatException will be thrown if o is not a valid number. + * Get the short value out of a primitive object. Note that + * NullPointerException will be thrown if o is null. Note that + * NumberFormatException will be thrown if o is not a valid number. */ - public static short getShort(Object o, PrimitiveObjectInspector oi) throws NumberFormatException { - return (short)getInt(o, oi); + public static short getShort(Object o, PrimitiveObjectInspector oi) + throws NumberFormatException { + return (short) getInt(o, oi); } - + /** - * Get the integer value out of a primitive object. - * Note that NullPointerException will be thrown if o is null. - * Note that NumberFormatException will be thrown if o is not a valid number. + * Get the integer value out of a primitive object. Note that + * NullPointerException will be thrown if o is null. Note that + * NumberFormatException will be thrown if o is not a valid number. */ - public static int getInt(Object o, PrimitiveObjectInspector oi) throws NumberFormatException { + public static int getInt(Object o, PrimitiveObjectInspector oi) + throws NumberFormatException { int result = 0; switch (oi.getPrimitiveCategory()) { - case VOID: { - result = 0; - break; + case VOID: { + result = 0; + break; + } + case BOOLEAN: { + result = (((BooleanObjectInspector) oi).get(o) ? 1 : 0); + break; + } + case BYTE: { + result = ((ByteObjectInspector) oi).get(o); + break; + } + case SHORT: { + result = ((ShortObjectInspector) oi).get(o); + break; + } + case INT: { + result = ((IntObjectInspector) oi).get(o); + break; + } + case LONG: { + result = (int) ((LongObjectInspector) oi).get(o); + break; + } + case FLOAT: { + result = (int) ((FloatObjectInspector) oi).get(o); + break; + } + case DOUBLE: { + result = (int) ((DoubleObjectInspector) oi).get(o); + break; + } + case STRING: { + StringObjectInspector soi = (StringObjectInspector) oi; + if (soi.preferWritable()) { + Text t = soi.getPrimitiveWritableObject(o); + result = LazyInteger.parseInt(t.getBytes(), 0, t.getLength()); + } else { + String s = soi.getPrimitiveJavaObject(o); + result = Integer.parseInt(s); } - case BOOLEAN: { - result = (((BooleanObjectInspector)oi).get(o) ? 1 : 0); - break; - } - case BYTE: { - result = ((ByteObjectInspector)oi).get(o); - break; - } - case SHORT: { - result = ((ShortObjectInspector)oi).get(o); - break; - } - case INT: { - result = ((IntObjectInspector)oi).get(o); - break; - } - case LONG: { - result = (int)((LongObjectInspector)oi).get(o); - break; - } - case FLOAT: { - result = (int)((FloatObjectInspector)oi).get(o); - break; - } - case DOUBLE: { - result = (int)((DoubleObjectInspector)oi).get(o); - break; - } - case STRING: { - StringObjectInspector soi = (StringObjectInspector)oi; - if (soi.preferWritable()) { - Text t = soi.getPrimitiveWritableObject(o); - result = LazyInteger.parseInt(t.getBytes(), 0, t.getLength()); - } else { - String s = soi.getPrimitiveJavaObject(o); - result = Integer.parseInt(s); - } - break; - } - default: { - throw new RuntimeException("Hive 2 Internal error: unknown type: " - + oi.getTypeName()); - } + break; } + default: { + throw new RuntimeException("Hive 2 Internal error: unknown type: " + + oi.getTypeName()); + } + } return result; } - /** - * Get the long value out of a primitive object. - * Note that NullPointerException will be thrown if o is null. - * Note that NumberFormatException will be thrown if o is not a valid number. + * Get the long value out of a primitive object. Note that + * NullPointerException will be thrown if o is null. Note that + * NumberFormatException will be thrown if o is not a valid number. */ - public static long getLong(Object o, PrimitiveObjectInspector oi) throws NumberFormatException { + public static long getLong(Object o, PrimitiveObjectInspector oi) + throws NumberFormatException { long result = 0; switch (oi.getPrimitiveCategory()) { - case VOID: { - result = 0; - break; + case VOID: { + result = 0; + break; + } + case BOOLEAN: { + result = (((BooleanObjectInspector) oi).get(o) ? 1 : 0); + break; + } + case BYTE: { + result = ((ByteObjectInspector) oi).get(o); + break; + } + case SHORT: { + result = ((ShortObjectInspector) oi).get(o); + break; + } + case INT: { + result = ((IntObjectInspector) oi).get(o); + break; + } + case LONG: { + result = ((LongObjectInspector) oi).get(o); + break; + } + case FLOAT: { + result = (long) ((FloatObjectInspector) oi).get(o); + break; + } + case DOUBLE: { + result = (long) ((DoubleObjectInspector) oi).get(o); + break; + } + case STRING: { + StringObjectInspector soi = (StringObjectInspector) oi; + if (soi.preferWritable()) { + Text t = soi.getPrimitiveWritableObject(o); + result = LazyLong.parseLong(t.getBytes(), 0, t.getLength()); + } else { + String s = soi.getPrimitiveJavaObject(o); + result = Long.parseLong(s); } - case BOOLEAN: { - result = (((BooleanObjectInspector)oi).get(o) ? 1 : 0); - break; - } - case BYTE: { - result = ((ByteObjectInspector)oi).get(o); - break; - } - case SHORT: { - result = ((ShortObjectInspector)oi).get(o); - break; - } - case INT: { - result = ((IntObjectInspector)oi).get(o); - break; - } - case LONG: { - result = ((LongObjectInspector)oi).get(o); - break; - } - case FLOAT: { - result = (long)((FloatObjectInspector)oi).get(o); - break; - } - case DOUBLE: { - result = (long)((DoubleObjectInspector)oi).get(o); - break; - } - case STRING: { - StringObjectInspector soi = (StringObjectInspector)oi; - if (soi.preferWritable()) { - Text t = soi.getPrimitiveWritableObject(o); - result = LazyLong.parseLong(t.getBytes(), 0, t.getLength()); - } else { - String s = soi.getPrimitiveJavaObject(o); - result = Long.parseLong(s); - } - break; - } - default: { - throw new RuntimeException("Hive 2 Internal error: unknown type: " - + oi.getTypeName()); - } + break; } + default: { + throw new RuntimeException("Hive 2 Internal error: unknown type: " + + oi.getTypeName()); + } + } return result; } - + /** - * Get the double value out of a primitive object. - * Note that NullPointerException will be thrown if o is null. - * Note that NumberFormatException will be thrown if o is not a valid number. + * Get the double value out of a primitive object. Note that + * NullPointerException will be thrown if o is null. Note that + * NumberFormatException will be thrown if o is not a valid number. */ - public static double getDouble(Object o, PrimitiveObjectInspector oi) throws NumberFormatException { + public static double getDouble(Object o, PrimitiveObjectInspector oi) + throws NumberFormatException { double result = 0; switch (oi.getPrimitiveCategory()) { - case VOID: { - result = 0; - break; - } - case BOOLEAN: { - result = (((BooleanObjectInspector)oi).get(o) ? 1 : 0); - break; - } - case BYTE: { - result = ((ByteObjectInspector)oi).get(o); - break; - } - case SHORT: { - result = ((ShortObjectInspector)oi).get(o); - break; - } - case INT: { - result = ((IntObjectInspector)oi).get(o); - break; - } - case LONG: { - result = ((LongObjectInspector)oi).get(o); - break; - } - case FLOAT: { - result = ((FloatObjectInspector)oi).get(o); - break; - } - case DOUBLE: { - result = ((DoubleObjectInspector)oi).get(o); - break; - } - case STRING: { - StringObjectInspector soi = (StringObjectInspector)oi; - String s = soi.getPrimitiveJavaObject(o); - result = Double.parseDouble(s); - break; - } - default: { - throw new RuntimeException("Hive 2 Internal error: unknown type: " - + oi.getTypeName()); - } + case VOID: { + result = 0; + break; } + case BOOLEAN: { + result = (((BooleanObjectInspector) oi).get(o) ? 1 : 0); + break; + } + case BYTE: { + result = ((ByteObjectInspector) oi).get(o); + break; + } + case SHORT: { + result = ((ShortObjectInspector) oi).get(o); + break; + } + case INT: { + result = ((IntObjectInspector) oi).get(o); + break; + } + case LONG: { + result = ((LongObjectInspector) oi).get(o); + break; + } + case FLOAT: { + result = ((FloatObjectInspector) oi).get(o); + break; + } + case DOUBLE: { + result = ((DoubleObjectInspector) oi).get(o); + break; + } + case STRING: { + StringObjectInspector soi = (StringObjectInspector) oi; + String s = soi.getPrimitiveJavaObject(o); + result = Double.parseDouble(s); + break; + } + default: { + throw new RuntimeException("Hive 2 Internal error: unknown type: " + + oi.getTypeName()); + } + } return result; - } + } /** - * Get the float value out of a primitive object. - * Note that NullPointerException will be thrown if o is null. - * Note that NumberFormatException will be thrown if o is not a valid number. + * Get the float value out of a primitive object. Note that + * NullPointerException will be thrown if o is null. Note that + * NumberFormatException will be thrown if o is not a valid number. */ - public static float getFloat(Object o, PrimitiveObjectInspector oi) throws NumberFormatException { - return (float)getDouble(o, oi); + public static float getFloat(Object o, PrimitiveObjectInspector oi) + throws NumberFormatException { + return (float) getDouble(o, oi); } - /** - * Get the String value out of a primitive object. - * Note that NullPointerException will be thrown if o is null. - * Note that NumberFormatException will be thrown if o is not a valid number. + * Get the String value out of a primitive object. Note that + * NullPointerException will be thrown if o is null. Note that + * NumberFormatException will be thrown if o is not a valid number. */ - public static String getString(Object o, PrimitiveObjectInspector oi) throws NumberFormatException { - + public static String getString(Object o, PrimitiveObjectInspector oi) + throws NumberFormatException { + if (o == null) { return null; } - + String result = null; switch (oi.getPrimitiveCategory()) { - case VOID: { - result = null; - break; - } - case BOOLEAN: { - result = String.valueOf((((BooleanObjectInspector)oi).get(o))); - break; - } - case BYTE: { - result = String.valueOf((((ByteObjectInspector)oi).get(o))); - break; - } - case SHORT: { - result = String.valueOf((((ShortObjectInspector)oi).get(o))); - break; - } - case INT: { - result = String.valueOf((((IntObjectInspector)oi).get(o))); - break; - } - case LONG: { - result = String.valueOf((((LongObjectInspector)oi).get(o))); - break; - } - case FLOAT: { - result = String.valueOf((((FloatObjectInspector)oi).get(o))); - break; - } - case DOUBLE: { - result = String.valueOf((((DoubleObjectInspector)oi).get(o))); - break; - } - case STRING: { - StringObjectInspector soi = (StringObjectInspector)oi; - result = soi.getPrimitiveJavaObject(o); - break; - } - default: { - throw new RuntimeException("Hive 2 Internal error: unknown type: " - + oi.getTypeName()); - } + case VOID: { + result = null; + break; } + case BOOLEAN: { + result = String.valueOf((((BooleanObjectInspector) oi).get(o))); + break; + } + case BYTE: { + result = String.valueOf((((ByteObjectInspector) oi).get(o))); + break; + } + case SHORT: { + result = String.valueOf((((ShortObjectInspector) oi).get(o))); + break; + } + case INT: { + result = String.valueOf((((IntObjectInspector) oi).get(o))); + break; + } + case LONG: { + result = String.valueOf((((LongObjectInspector) oi).get(o))); + break; + } + case FLOAT: { + result = String.valueOf((((FloatObjectInspector) oi).get(o))); + break; + } + case DOUBLE: { + result = String.valueOf((((DoubleObjectInspector) oi).get(o))); + break; + } + case STRING: { + StringObjectInspector soi = (StringObjectInspector) oi; + result = soi.getPrimitiveJavaObject(o); + break; + } + default: { + throw new RuntimeException("Hive 2 Internal error: unknown type: " + + oi.getTypeName()); + } + } return result; - } - + } + } Index: serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/IntObjectInspector.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/IntObjectInspector.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/IntObjectInspector.java (working copy) @@ -19,12 +19,11 @@ import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; - /** * A IntObjectInspector inspects an Object representing an Integer. */ public interface IntObjectInspector extends PrimitiveObjectInspector { - + /** * Get the int data. */ Index: serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/JavaIntObjectInspector.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/JavaIntObjectInspector.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/JavaIntObjectInspector.java (working copy) @@ -19,25 +19,24 @@ import org.apache.hadoop.io.IntWritable; - /** * A JavaIntObjectInspector inspects a Java Integer Object. */ -public class JavaIntObjectInspector extends AbstractPrimitiveJavaObjectInspector -implements SettableIntObjectInspector{ +public class JavaIntObjectInspector extends + AbstractPrimitiveJavaObjectInspector implements SettableIntObjectInspector { JavaIntObjectInspector() { super(PrimitiveObjectInspectorUtils.intTypeEntry); } - + @Override public Object getPrimitiveWritableObject(Object o) { - return o == null ? null : new IntWritable(((Integer)o).intValue()); + return o == null ? null : new IntWritable(((Integer) o).intValue()); } @Override public int get(Object o) { - return ((Integer)o).intValue(); + return ((Integer) o).intValue(); } @Override Index: serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/AbstractPrimitiveObjectInspector.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/AbstractPrimitiveObjectInspector.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/AbstractPrimitiveObjectInspector.java (working copy) @@ -21,9 +21,11 @@ import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils.PrimitiveTypeEntry; /** - * An AbstractPrimitiveObjectInspector is based on ObjectInspectorUtils.PrimitiveTypeEntry. + * An AbstractPrimitiveObjectInspector is based on + * ObjectInspectorUtils.PrimitiveTypeEntry. */ -public abstract class AbstractPrimitiveObjectInspector implements PrimitiveObjectInspector { +public abstract class AbstractPrimitiveObjectInspector implements + PrimitiveObjectInspector { PrimitiveTypeEntry typeEntry; @@ -31,11 +33,12 @@ * Construct a AbstractPrimitiveObjectInspector. */ protected AbstractPrimitiveObjectInspector(PrimitiveTypeEntry typeEntry) { - this.typeEntry = typeEntry; + this.typeEntry = typeEntry; } /** - * Return the associated Java primitive class for this primitive ObjectInspector. + * Return the associated Java primitive class for this primitive + * ObjectInspector. */ @Override public Class getJavaPrimitiveClass() { @@ -43,7 +46,8 @@ } /** - * Return the associated primitive category for this primitive ObjectInspector. + * Return the associated primitive category for this primitive + * ObjectInspector. */ @Override public PrimitiveCategory getPrimitiveCategory() { @@ -51,7 +55,8 @@ } /** - * Return the associated primitive Writable class for this primitive ObjectInspector. + * Return the associated primitive Writable class for this primitive + * ObjectInspector. */ @Override public Class getPrimitiveWritableClass() { Index: serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableByteObjectInspector.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableByteObjectInspector.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableByteObjectInspector.java (working copy) @@ -18,34 +18,33 @@ package org.apache.hadoop.hive.serde2.objectinspector.primitive; import org.apache.hadoop.hive.serde2.io.ByteWritable; -import org.apache.hadoop.io.BooleanWritable; - /** * A WritableByteObjectInspector inspects a ByteWritable Object. */ -public class WritableByteObjectInspector extends AbstractPrimitiveWritableObjectInspector -implements SettableByteObjectInspector{ +public class WritableByteObjectInspector extends + AbstractPrimitiveWritableObjectInspector implements + SettableByteObjectInspector { WritableByteObjectInspector() { super(PrimitiveObjectInspectorUtils.byteTypeEntry); } - + @Override public byte get(Object o) { - return ((ByteWritable)o).get(); + return ((ByteWritable) o).get(); } @Override public Object copyObject(Object o) { - return o == null ? null : new ByteWritable(((ByteWritable)o).get()); + return o == null ? null : new ByteWritable(((ByteWritable) o).get()); } @Override public Object getPrimitiveJavaObject(Object o) { - return o == null ? null : Byte.valueOf(((ByteWritable)o).get()); + return o == null ? null : Byte.valueOf(((ByteWritable) o).get()); } - + @Override public Object create(byte value) { return new ByteWritable(value); @@ -53,7 +52,7 @@ @Override public Object set(Object o, byte value) { - ((ByteWritable)o).set(value); + ((ByteWritable) o).set(value); return o; } } Index: serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/AbstractPrimitiveJavaObjectInspector.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/AbstractPrimitiveJavaObjectInspector.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/AbstractPrimitiveJavaObjectInspector.java (working copy) @@ -20,14 +20,15 @@ import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils.PrimitiveTypeEntry; /** - * An AbstractJavaPrimitiveObjectInspector for a Java object. + * An AbstractJavaPrimitiveObjectInspector for a Java object. */ -public abstract class AbstractPrimitiveJavaObjectInspector extends AbstractPrimitiveObjectInspector { +public abstract class AbstractPrimitiveJavaObjectInspector extends + AbstractPrimitiveObjectInspector { protected AbstractPrimitiveJavaObjectInspector(PrimitiveTypeEntry typeEntry) { super(typeEntry); } - + @Override public Object getPrimitiveJavaObject(Object o) { return o; @@ -37,7 +38,7 @@ public Object copyObject(Object o) { return o; } - + @Override public boolean preferWritable() { return false; Index: serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/SettableDoubleObjectInspector.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/SettableDoubleObjectInspector.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/SettableDoubleObjectInspector.java (working copy) @@ -17,7 +17,6 @@ */ package org.apache.hadoop.hive.serde2.objectinspector.primitive; - /** * A SettableDoubleObjectInspector can set a double value to an object. */ @@ -26,11 +25,11 @@ /** * Set the object with the value. Return the object that has the new value. * - * In most cases the returned value should be the same as o, but in case - * o is unmodifiable, this will return a new object with new value. + * In most cases the returned value should be the same as o, but in case o is + * unmodifiable, this will return a new object with new value. */ public Object set(Object o, double value); - + /** * Create an object with the value. */ Index: serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableFloatObjectInspector.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableFloatObjectInspector.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableFloatObjectInspector.java (working copy) @@ -17,32 +17,32 @@ */ package org.apache.hadoop.hive.serde2.objectinspector.primitive; -import org.apache.hadoop.hive.serde2.io.DoubleWritable; import org.apache.hadoop.io.FloatWritable; /** * A FloatObjectInspector inspects a FloatWritable Object. */ -public class WritableFloatObjectInspector extends AbstractPrimitiveWritableObjectInspector -implements SettableFloatObjectInspector{ +public class WritableFloatObjectInspector extends + AbstractPrimitiveWritableObjectInspector implements + SettableFloatObjectInspector { WritableFloatObjectInspector() { super(PrimitiveObjectInspectorUtils.floatTypeEntry); } - + @Override public float get(Object o) { - return ((FloatWritable)o).get(); + return ((FloatWritable) o).get(); } @Override public Object copyObject(Object o) { - return o == null ? null : new FloatWritable(((FloatWritable)o).get()); + return o == null ? null : new FloatWritable(((FloatWritable) o).get()); } @Override public Object getPrimitiveJavaObject(Object o) { - return o == null ? null : Float.valueOf(((FloatWritable)o).get()); + return o == null ? null : Float.valueOf(((FloatWritable) o).get()); } @Override @@ -52,7 +52,7 @@ @Override public Object set(Object o, float value) { - ((FloatWritable)o).set(value); + ((FloatWritable) o).set(value); return o; } } Index: serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorFactory.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorFactory.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorFactory.java (working copy) @@ -22,117 +22,109 @@ import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableBooleanObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils.PrimitiveTypeEntry; import org.apache.hadoop.io.Writable; - /** - * PrimitiveObjectInspectorFactory is the primary way to create new PrimitiveObjectInspector - * instances. + * PrimitiveObjectInspectorFactory is the primary way to create new + * PrimitiveObjectInspector instances. * - * The reason of having caches here is that ObjectInspector is because ObjectInspectors do - * not have an internal state - so ObjectInspectors with the same construction parameters should - * result in exactly the same ObjectInspector. + * The reason of having caches here is that ObjectInspector is because + * ObjectInspectors do not have an internal state - so ObjectInspectors with the + * same construction parameters should result in exactly the same + * ObjectInspector. */ public class PrimitiveObjectInspectorFactory { - public final static JavaBooleanObjectInspector javaBooleanObjectInspector= new JavaBooleanObjectInspector(); - public final static JavaByteObjectInspector javaByteObjectInspector= new JavaByteObjectInspector(); - public final static JavaShortObjectInspector javaShortObjectInspector= new JavaShortObjectInspector(); - public final static JavaIntObjectInspector javaIntObjectInspector= new JavaIntObjectInspector(); - public final static JavaLongObjectInspector javaLongObjectInspector= new JavaLongObjectInspector(); - public final static JavaFloatObjectInspector javaFloatObjectInspector= new JavaFloatObjectInspector(); - public final static JavaDoubleObjectInspector javaDoubleObjectInspector= new JavaDoubleObjectInspector(); - public final static JavaStringObjectInspector javaStringObjectInspector= new JavaStringObjectInspector(); - public final static JavaVoidObjectInspector javaVoidObjectInspector= new JavaVoidObjectInspector(); + public final static JavaBooleanObjectInspector javaBooleanObjectInspector = new JavaBooleanObjectInspector(); + public final static JavaByteObjectInspector javaByteObjectInspector = new JavaByteObjectInspector(); + public final static JavaShortObjectInspector javaShortObjectInspector = new JavaShortObjectInspector(); + public final static JavaIntObjectInspector javaIntObjectInspector = new JavaIntObjectInspector(); + public final static JavaLongObjectInspector javaLongObjectInspector = new JavaLongObjectInspector(); + public final static JavaFloatObjectInspector javaFloatObjectInspector = new JavaFloatObjectInspector(); + public final static JavaDoubleObjectInspector javaDoubleObjectInspector = new JavaDoubleObjectInspector(); + public final static JavaStringObjectInspector javaStringObjectInspector = new JavaStringObjectInspector(); + public final static JavaVoidObjectInspector javaVoidObjectInspector = new JavaVoidObjectInspector(); - public final static WritableBooleanObjectInspector writableBooleanObjectInspector= new WritableBooleanObjectInspector(); - public final static WritableByteObjectInspector writableByteObjectInspector= new WritableByteObjectInspector(); - public final static WritableShortObjectInspector writableShortObjectInspector= new WritableShortObjectInspector(); - public final static WritableIntObjectInspector writableIntObjectInspector= new WritableIntObjectInspector(); - public final static WritableLongObjectInspector writableLongObjectInspector= new WritableLongObjectInspector(); - public final static WritableFloatObjectInspector writableFloatObjectInspector= new WritableFloatObjectInspector(); - public final static WritableDoubleObjectInspector writableDoubleObjectInspector= new WritableDoubleObjectInspector(); - public final static WritableStringObjectInspector writableStringObjectInspector= new WritableStringObjectInspector(); - public final static WritableVoidObjectInspector writableVoidObjectInspector= new WritableVoidObjectInspector(); - - - private static HashMap - cachedPrimitiveWritableInspectorCache = - new HashMap(); + public final static WritableBooleanObjectInspector writableBooleanObjectInspector = new WritableBooleanObjectInspector(); + public final static WritableByteObjectInspector writableByteObjectInspector = new WritableByteObjectInspector(); + public final static WritableShortObjectInspector writableShortObjectInspector = new WritableShortObjectInspector(); + public final static WritableIntObjectInspector writableIntObjectInspector = new WritableIntObjectInspector(); + public final static WritableLongObjectInspector writableLongObjectInspector = new WritableLongObjectInspector(); + public final static WritableFloatObjectInspector writableFloatObjectInspector = new WritableFloatObjectInspector(); + public final static WritableDoubleObjectInspector writableDoubleObjectInspector = new WritableDoubleObjectInspector(); + public final static WritableStringObjectInspector writableStringObjectInspector = new WritableStringObjectInspector(); + public final static WritableVoidObjectInspector writableVoidObjectInspector = new WritableVoidObjectInspector(); + + private static HashMap cachedPrimitiveWritableInspectorCache = new HashMap(); static { - cachedPrimitiveWritableInspectorCache.put(PrimitiveCategory.BOOLEAN, + cachedPrimitiveWritableInspectorCache.put(PrimitiveCategory.BOOLEAN, writableBooleanObjectInspector); - cachedPrimitiveWritableInspectorCache.put(PrimitiveCategory.BYTE, + cachedPrimitiveWritableInspectorCache.put(PrimitiveCategory.BYTE, writableByteObjectInspector); - cachedPrimitiveWritableInspectorCache.put(PrimitiveCategory.SHORT, + cachedPrimitiveWritableInspectorCache.put(PrimitiveCategory.SHORT, writableShortObjectInspector); - cachedPrimitiveWritableInspectorCache.put(PrimitiveCategory.INT, + cachedPrimitiveWritableInspectorCache.put(PrimitiveCategory.INT, writableIntObjectInspector); - cachedPrimitiveWritableInspectorCache.put(PrimitiveCategory.LONG, + cachedPrimitiveWritableInspectorCache.put(PrimitiveCategory.LONG, writableLongObjectInspector); - cachedPrimitiveWritableInspectorCache.put(PrimitiveCategory.FLOAT, + cachedPrimitiveWritableInspectorCache.put(PrimitiveCategory.FLOAT, writableFloatObjectInspector); - cachedPrimitiveWritableInspectorCache.put(PrimitiveCategory.DOUBLE, + cachedPrimitiveWritableInspectorCache.put(PrimitiveCategory.DOUBLE, writableDoubleObjectInspector); - cachedPrimitiveWritableInspectorCache.put(PrimitiveCategory.STRING, + cachedPrimitiveWritableInspectorCache.put(PrimitiveCategory.STRING, writableStringObjectInspector); - cachedPrimitiveWritableInspectorCache.put(PrimitiveCategory.VOID, + cachedPrimitiveWritableInspectorCache.put(PrimitiveCategory.VOID, writableVoidObjectInspector); } - - private static HashMap - cachedPrimitiveJavaInspectorCache = - new HashMap(); + private static HashMap cachedPrimitiveJavaInspectorCache = new HashMap(); static { - cachedPrimitiveJavaInspectorCache.put(PrimitiveCategory.BOOLEAN, + cachedPrimitiveJavaInspectorCache.put(PrimitiveCategory.BOOLEAN, javaBooleanObjectInspector); - cachedPrimitiveJavaInspectorCache.put(PrimitiveCategory.BYTE, + cachedPrimitiveJavaInspectorCache.put(PrimitiveCategory.BYTE, javaByteObjectInspector); - cachedPrimitiveJavaInspectorCache.put(PrimitiveCategory.SHORT, + cachedPrimitiveJavaInspectorCache.put(PrimitiveCategory.SHORT, javaShortObjectInspector); - cachedPrimitiveJavaInspectorCache.put(PrimitiveCategory.INT, + cachedPrimitiveJavaInspectorCache.put(PrimitiveCategory.INT, javaIntObjectInspector); - cachedPrimitiveJavaInspectorCache.put(PrimitiveCategory.LONG, + cachedPrimitiveJavaInspectorCache.put(PrimitiveCategory.LONG, javaLongObjectInspector); - cachedPrimitiveJavaInspectorCache.put(PrimitiveCategory.FLOAT, + cachedPrimitiveJavaInspectorCache.put(PrimitiveCategory.FLOAT, javaFloatObjectInspector); - cachedPrimitiveJavaInspectorCache.put(PrimitiveCategory.DOUBLE, + cachedPrimitiveJavaInspectorCache.put(PrimitiveCategory.DOUBLE, javaDoubleObjectInspector); - cachedPrimitiveJavaInspectorCache.put(PrimitiveCategory.STRING, + cachedPrimitiveJavaInspectorCache.put(PrimitiveCategory.STRING, javaStringObjectInspector); - cachedPrimitiveJavaInspectorCache.put(PrimitiveCategory.VOID, + cachedPrimitiveJavaInspectorCache.put(PrimitiveCategory.VOID, javaVoidObjectInspector); } - - + /** - * Returns the PrimitiveWritableObjectInspector for the PrimitiveCategory. + * Returns the PrimitiveWritableObjectInspector for the PrimitiveCategory. + * * @param primitiveCategory */ - public static AbstractPrimitiveWritableObjectInspector - getPrimitiveWritableObjectInspector( - PrimitiveCategory primitiveCategory) { - AbstractPrimitiveWritableObjectInspector result = - cachedPrimitiveWritableInspectorCache.get(primitiveCategory); + public static AbstractPrimitiveWritableObjectInspector getPrimitiveWritableObjectInspector( + PrimitiveCategory primitiveCategory) { + AbstractPrimitiveWritableObjectInspector result = cachedPrimitiveWritableInspectorCache + .get(primitiveCategory); if (result == null) { throw new RuntimeException("Internal error: Cannot find ObjectInspector " + " for " + primitiveCategory); } return result; } - + /** - * Returns the PrimitiveJavaObjectInspector for the PrimitiveCategory. + * Returns the PrimitiveJavaObjectInspector for the PrimitiveCategory. + * * @param primitiveCategory */ - public static AbstractPrimitiveJavaObjectInspector - getPrimitiveJavaObjectInspector( - PrimitiveCategory primitiveCategory) { - AbstractPrimitiveJavaObjectInspector result = - cachedPrimitiveJavaInspectorCache.get(primitiveCategory); + public static AbstractPrimitiveJavaObjectInspector getPrimitiveJavaObjectInspector( + PrimitiveCategory primitiveCategory) { + AbstractPrimitiveJavaObjectInspector result = cachedPrimitiveJavaInspectorCache + .get(primitiveCategory); if (result == null) { throw new RuntimeException("Internal error: Cannot find ObjectInspector " + " for " + primitiveCategory); @@ -141,14 +133,14 @@ } /** - * Returns an ObjectInspector for a primitive Class. - * The Class can be a Hive Writable class, or a Java Primitive Class. + * Returns an ObjectInspector for a primitive Class. The Class can be a Hive + * Writable class, or a Java Primitive Class. * - * A runtimeException will be thrown if the class is not recognized - * as a primitive type by Hive. + * A runtimeException will be thrown if the class is not recognized as a + * primitive type by Hive. */ - public static PrimitiveObjectInspector - getPrimitiveObjectInspectorFromClass(Class c) { + public static PrimitiveObjectInspector getPrimitiveObjectInspectorFromClass( + Class c) { if (Writable.class.isAssignableFrom(c)) { // It is a writable class PrimitiveTypeEntry te = PrimitiveObjectInspectorUtils @@ -169,6 +161,5 @@ .getPrimitiveJavaObjectInspector(te.primitiveCategory); } } - - + } Index: serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorConverter.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorConverter.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorConverter.java (working copy) @@ -34,8 +34,9 @@ PrimitiveObjectInspector inputOI; SettableBooleanObjectInspector outputOI; Object r; - - public BooleanConverter(PrimitiveObjectInspector inputOI, SettableBooleanObjectInspector outputOI) { + + public BooleanConverter(PrimitiveObjectInspector inputOI, + SettableBooleanObjectInspector outputOI) { this.inputOI = inputOI; this.outputOI = outputOI; r = outputOI.create(false); @@ -47,7 +48,8 @@ return null; } try { - return outputOI.set(r, PrimitiveObjectInspectorUtils.getBoolean(input, inputOI)); + return outputOI.set(r, PrimitiveObjectInspectorUtils.getBoolean(input, + inputOI)); } catch (NumberFormatException e) { return null; } @@ -61,11 +63,12 @@ PrimitiveObjectInspector inputOI; SettableByteObjectInspector outputOI; Object r; - - public ByteConverter(PrimitiveObjectInspector inputOI, SettableByteObjectInspector outputOI) { + + public ByteConverter(PrimitiveObjectInspector inputOI, + SettableByteObjectInspector outputOI) { this.inputOI = inputOI; this.outputOI = outputOI; - r = outputOI.create((byte)0); + r = outputOI.create((byte) 0); } @Override @@ -74,7 +77,8 @@ return null; } try { - return outputOI.set(r, PrimitiveObjectInspectorUtils.getByte(input, inputOI)); + return outputOI.set(r, PrimitiveObjectInspectorUtils.getByte(input, + inputOI)); } catch (NumberFormatException e) { return null; } @@ -88,11 +92,12 @@ PrimitiveObjectInspector inputOI; SettableShortObjectInspector outputOI; Object r; - - public ShortConverter(PrimitiveObjectInspector inputOI, SettableShortObjectInspector outputOI) { + + public ShortConverter(PrimitiveObjectInspector inputOI, + SettableShortObjectInspector outputOI) { this.inputOI = inputOI; this.outputOI = outputOI; - r = outputOI.create((short)0); + r = outputOI.create((short) 0); } @Override @@ -101,7 +106,8 @@ return null; } try { - return outputOI.set(r, PrimitiveObjectInspectorUtils.getShort(input, inputOI)); + return outputOI.set(r, PrimitiveObjectInspectorUtils.getShort(input, + inputOI)); } catch (NumberFormatException e) { return null; } @@ -115,11 +121,12 @@ PrimitiveObjectInspector inputOI; SettableIntObjectInspector outputOI; Object r; - - public IntConverter(PrimitiveObjectInspector inputOI, SettableIntObjectInspector outputOI) { + + public IntConverter(PrimitiveObjectInspector inputOI, + SettableIntObjectInspector outputOI) { this.inputOI = inputOI; this.outputOI = outputOI; - r = outputOI.create((int)0); + r = outputOI.create(0); } @Override @@ -128,7 +135,8 @@ return null; } try { - return outputOI.set(r, PrimitiveObjectInspectorUtils.getInt(input, inputOI)); + return outputOI.set(r, PrimitiveObjectInspectorUtils.getInt(input, + inputOI)); } catch (NumberFormatException e) { return null; } @@ -142,11 +150,12 @@ PrimitiveObjectInspector inputOI; SettableLongObjectInspector outputOI; Object r; - - public LongConverter(PrimitiveObjectInspector inputOI, SettableLongObjectInspector outputOI) { + + public LongConverter(PrimitiveObjectInspector inputOI, + SettableLongObjectInspector outputOI) { this.inputOI = inputOI; this.outputOI = outputOI; - r = outputOI.create((long)0); + r = outputOI.create(0); } @Override @@ -155,13 +164,14 @@ return null; } try { - return outputOI.set(r, PrimitiveObjectInspectorUtils.getLong(input, inputOI)); + return outputOI.set(r, PrimitiveObjectInspectorUtils.getLong(input, + inputOI)); } catch (NumberFormatException e) { return null; } } } - + /** * A converter for the float type. */ @@ -169,11 +179,12 @@ PrimitiveObjectInspector inputOI; SettableFloatObjectInspector outputOI; Object r; - - public FloatConverter(PrimitiveObjectInspector inputOI, SettableFloatObjectInspector outputOI) { + + public FloatConverter(PrimitiveObjectInspector inputOI, + SettableFloatObjectInspector outputOI) { this.inputOI = inputOI; this.outputOI = outputOI; - r = outputOI.create((float)0); + r = outputOI.create(0); } @Override @@ -182,13 +193,14 @@ return null; } try { - return outputOI.set(r, PrimitiveObjectInspectorUtils.getFloat(input, inputOI)); + return outputOI.set(r, PrimitiveObjectInspectorUtils.getFloat(input, + inputOI)); } catch (NumberFormatException e) { return null; } } } - + /** * A converter for the double type. */ @@ -196,11 +208,12 @@ PrimitiveObjectInspector inputOI; SettableDoubleObjectInspector outputOI; Object r; - - public DoubleConverter(PrimitiveObjectInspector inputOI, SettableDoubleObjectInspector outputOI) { + + public DoubleConverter(PrimitiveObjectInspector inputOI, + SettableDoubleObjectInspector outputOI) { this.inputOI = inputOI; this.outputOI = outputOI; - r = outputOI.create((double)0); + r = outputOI.create(0); } @Override @@ -209,92 +222,98 @@ return null; } try { - return outputOI.set(r, PrimitiveObjectInspectorUtils.getDouble(input, inputOI)); + return outputOI.set(r, PrimitiveObjectInspectorUtils.getDouble(input, + inputOI)); } catch (NumberFormatException e) { return null; } } } - + /** - * A helper class to convert any primitive to Text. + * A helper class to convert any primitive to Text. */ public static class TextConverter implements Converter { PrimitiveObjectInspector inputOI; Text t = new Text(); ByteStream.Output out = new ByteStream.Output(); - - static byte[] trueBytes = {'T', 'R', 'U', 'E'}; - static byte[] falseBytes = {'F', 'A', 'L', 'S', 'E'}; - + + static byte[] trueBytes = { 'T', 'R', 'U', 'E' }; + static byte[] falseBytes = { 'F', 'A', 'L', 'S', 'E' }; + public TextConverter(PrimitiveObjectInspector inputOI) { // The output ObjectInspector is writableStringObjectInspector. this.inputOI = inputOI; } - - public Text convert(Object input) { + + public Text convert(Object input) { if (input == null) { return null; } - - switch(inputOI.getPrimitiveCategory()) { - case VOID: { - return null; - } - case BOOLEAN: { - t.set(((BooleanObjectInspector)inputOI).get(input) ? trueBytes : falseBytes); - return t; - } - case BYTE: { - out.reset(); - LazyInteger.writeUTF8NoException(out, ((ByteObjectInspector)inputOI).get(input)); - t.set(out.getData(), 0, out.getCount()); - return t; - } - case SHORT: { - out.reset(); - LazyInteger.writeUTF8NoException(out, ((ShortObjectInspector)inputOI).get(input)); - t.set(out.getData(), 0, out.getCount()); - return t; - } - case INT: { - out.reset(); - LazyInteger.writeUTF8NoException(out, ((IntObjectInspector)inputOI).get(input)); - t.set(out.getData(), 0, out.getCount()); - return t; - } - case LONG:{ - out.reset(); - LazyLong.writeUTF8NoException(out, ((LongObjectInspector)inputOI).get(input)); - t.set(out.getData(), 0, out.getCount()); - return t; - } - case FLOAT: { - t.set(String.valueOf(((FloatObjectInspector)inputOI).get(input))); - return t; - } - case DOUBLE: { - t.set(String.valueOf(((DoubleObjectInspector)inputOI).get(input))); - return t; - } - case STRING: { - t.set(((StringObjectInspector)inputOI).getPrimitiveJavaObject(input)); - return t; - } - default: { - throw new RuntimeException("Hive 2 Internal error: type = " - + inputOI.getTypeName()); - } + + switch (inputOI.getPrimitiveCategory()) { + case VOID: { + return null; } + case BOOLEAN: { + t.set(((BooleanObjectInspector) inputOI).get(input) ? trueBytes + : falseBytes); + return t; + } + case BYTE: { + out.reset(); + LazyInteger.writeUTF8NoException(out, ((ByteObjectInspector) inputOI) + .get(input)); + t.set(out.getData(), 0, out.getCount()); + return t; + } + case SHORT: { + out.reset(); + LazyInteger.writeUTF8NoException(out, ((ShortObjectInspector) inputOI) + .get(input)); + t.set(out.getData(), 0, out.getCount()); + return t; + } + case INT: { + out.reset(); + LazyInteger.writeUTF8NoException(out, ((IntObjectInspector) inputOI) + .get(input)); + t.set(out.getData(), 0, out.getCount()); + return t; + } + case LONG: { + out.reset(); + LazyLong.writeUTF8NoException(out, ((LongObjectInspector) inputOI) + .get(input)); + t.set(out.getData(), 0, out.getCount()); + return t; + } + case FLOAT: { + t.set(String.valueOf(((FloatObjectInspector) inputOI).get(input))); + return t; + } + case DOUBLE: { + t.set(String.valueOf(((DoubleObjectInspector) inputOI).get(input))); + return t; + } + case STRING: { + t.set(((StringObjectInspector) inputOI).getPrimitiveJavaObject(input)); + return t; + } + default: { + throw new RuntimeException("Hive 2 Internal error: type = " + + inputOI.getTypeName()); + } + } } } - + /** - * A helper class to convert any primitive to String. + * A helper class to convert any primitive to String. */ public static class StringConverter implements Converter { PrimitiveObjectInspector inputOI; - + public StringConverter(PrimitiveObjectInspector inputOI) { // The output ObjectInspector is writableStringObjectInspector. this.inputOI = inputOI; @@ -304,6 +323,6 @@ public Object convert(Object input) { return PrimitiveObjectInspectorUtils.getString(input, inputOI); } - } - + } + } Index: serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableStringObjectInspector.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableStringObjectInspector.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableStringObjectInspector.java (working copy) @@ -19,12 +19,12 @@ import org.apache.hadoop.io.Text; - /** * A WritableStringObjectInspector inspects a Text Object. */ -public class WritableStringObjectInspector extends AbstractPrimitiveWritableObjectInspector -implements SettableStringObjectInspector{ +public class WritableStringObjectInspector extends + AbstractPrimitiveWritableObjectInspector implements + SettableStringObjectInspector { WritableStringObjectInspector() { super(PrimitiveObjectInspectorUtils.stringTypeEntry); @@ -32,17 +32,17 @@ @Override public Object copyObject(Object o) { - return o == null ? null : new Text((Text)o); + return o == null ? null : new Text((Text) o); } @Override public Text getPrimitiveWritableObject(Object o) { - return o == null ? null : (Text)o; + return o == null ? null : (Text) o; } - + @Override public String getPrimitiveJavaObject(Object o) { - return o == null ? null : ((Text)o).toString(); + return o == null ? null : ((Text) o).toString(); } @Override @@ -65,7 +65,7 @@ @Override public Object set(Object o, Text value) { - Text r = (Text)o; + Text r = (Text) o; if (value != null) { r.set(value); } @@ -74,7 +74,7 @@ @Override public Object set(Object o, String value) { - Text r = (Text)o; + Text r = (Text) o; if (value != null) { r.set(value); } Index: serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/VoidObjectInspector.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/VoidObjectInspector.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/VoidObjectInspector.java (working copy) @@ -19,10 +19,9 @@ import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; - /** * A VoidObjectInspector can inspect a void object. */ public interface VoidObjectInspector extends PrimitiveObjectInspector { - + } Index: serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/SettableBooleanObjectInspector.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/SettableBooleanObjectInspector.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/SettableBooleanObjectInspector.java (working copy) @@ -17,7 +17,6 @@ */ package org.apache.hadoop.hive.serde2.objectinspector.primitive; - /** * A SettableBooleanObjectInspector can set a boolean value to an object. */ @@ -26,11 +25,11 @@ /** * Set the object with the value. Return the object that has the new value. * - * In most cases the returned value should be the same as o, but in case - * o is unmodifiable, this will return a new object with new value. + * In most cases the returned value should be the same as o, but in case o is + * unmodifiable, this will return a new object with new value. */ public Object set(Object o, boolean value); - + /** * Create an object with the value. */ Index: serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/JavaVoidObjectInspector.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/JavaVoidObjectInspector.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/JavaVoidObjectInspector.java (working copy) @@ -19,17 +19,16 @@ import org.apache.hadoop.io.NullWritable; - /** * A JavaVoidObjectInspector inspects a Java Void Object. */ -public class JavaVoidObjectInspector extends AbstractPrimitiveJavaObjectInspector -implements VoidObjectInspector{ +public class JavaVoidObjectInspector extends + AbstractPrimitiveJavaObjectInspector implements VoidObjectInspector { JavaVoidObjectInspector() { super(PrimitiveObjectInspectorUtils.voidTypeEntry); } - + @Override public Object getPrimitiveWritableObject(Object o) { return NullWritable.get(); Index: serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/DoubleObjectInspector.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/DoubleObjectInspector.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/DoubleObjectInspector.java (working copy) @@ -19,12 +19,11 @@ import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; - /** * A DoubleObjectInspector inspects an Object representing a Double. */ public interface DoubleObjectInspector extends PrimitiveObjectInspector { - + /** * Get the double data. */ Index: serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/JavaDoubleObjectInspector.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/JavaDoubleObjectInspector.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/JavaDoubleObjectInspector.java (working copy) @@ -19,25 +19,25 @@ import org.apache.hadoop.hive.serde2.io.DoubleWritable; - /** * A JavaDoubleObjectInspector inspects a Java Double Object. */ -public class JavaDoubleObjectInspector extends AbstractPrimitiveJavaObjectInspector -implements SettableDoubleObjectInspector{ +public class JavaDoubleObjectInspector extends + AbstractPrimitiveJavaObjectInspector implements + SettableDoubleObjectInspector { JavaDoubleObjectInspector() { super(PrimitiveObjectInspectorUtils.doubleTypeEntry); } - + @Override public Object getPrimitiveWritableObject(Object o) { - return o == null ? null : new DoubleWritable(((Double)o).doubleValue()); + return o == null ? null : new DoubleWritable(((Double) o).doubleValue()); } @Override public double get(Object o) { - return ((Double)o).doubleValue(); + return ((Double) o).doubleValue(); } @Override Index: serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/BooleanObjectInspector.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/BooleanObjectInspector.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/BooleanObjectInspector.java (working copy) @@ -19,12 +19,11 @@ import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; - /** * A BooleanObjectInspector inspects an Object representing a Boolean. */ public interface BooleanObjectInspector extends PrimitiveObjectInspector { - + /** * Get the boolean data. */ Index: serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StructField.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StructField.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StructField.java (working copy) @@ -21,19 +21,19 @@ /** * StructField is an empty interface. * - * Classes implementing this interface are considered to represent - * a field of a struct for this serde package. + * Classes implementing this interface are considered to represent a field of a + * struct for this serde package. */ public interface StructField { /** - * Get the name of the field. The name should be always in lower-case. + * Get the name of the field. The name should be always in lower-case. */ String getFieldName(); - + /** * Get the ObjectInspector for the field. */ ObjectInspector getFieldObjectInspector(); - + } Index: serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ListObjectInspector.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ListObjectInspector.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ListObjectInspector.java (working copy) @@ -19,28 +19,30 @@ import java.util.List; - public interface ListObjectInspector extends ObjectInspector { // ** Methods that does not need a data object ** public ObjectInspector getListElementObjectInspector(); // ** Methods that need a data object ** - /** returns null for null list, out-of-the-range index. + /** + * returns null for null list, out-of-the-range index. */ public Object getListElement(Object data, int index); - /** returns -1 for data = null. + /** + * returns -1 for data = null. */ public int getListLength(Object data); - - /** returns null for data = null. - * - * Note: This method should not return a List object that is reused by the - * same ListObjectInspector, because it's possible that the same - * ListObjectInspector will be used in multiple places in the code. - * - * However it's OK if the List object is part of the Object data. + + /** + * returns null for data = null. + * + * Note: This method should not return a List object that is reused by the + * same ListObjectInspector, because it's possible that the same + * ListObjectInspector will be used in multiple places in the code. + * + * However it's OK if the List object is part of the Object data. */ public List getList(Object data); Index: serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/MapObjectInspector.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/MapObjectInspector.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/MapObjectInspector.java (working copy) @@ -19,7 +19,6 @@ import java.util.Map; - public interface MapObjectInspector extends ObjectInspector { // ** Methods that does not need a data object ** @@ -31,19 +30,21 @@ // ** Methods that need a data object ** // In this function, key has to be of the same structure as the Map expects. // Most cases key will be primitive type, so it's OK. - // In rare cases that key is not primitive, the user is responsible for defining + // In rare cases that key is not primitive, the user is responsible for + // defining // the hashCode() and equals() methods of the key class. public Object getMapValueElement(Object data, Object key); - /** returns null for data = null. - * - * Note: This method should not return a Map object that is reused by the - * same MapObjectInspector, because it's possible that the same - * MapObjectInspector will be used in multiple places in the code. - * - * However it's OK if the Map object is part of the Object data. + /** + * returns null for data = null. + * + * Note: This method should not return a Map object that is reused by the same + * MapObjectInspector, because it's possible that the same MapObjectInspector + * will be used in multiple places in the code. + * + * However it's OK if the Map object is part of the Object data. */ - public Map getMap(Object data); + public Map getMap(Object data); /** * returns -1 for NULL map. Index: serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspector.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspector.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspector.java (working copy) @@ -21,18 +21,19 @@ /** * ObjectInspector helps us to look into the internal structure of a complex * object. - * + * * A (probably configured) ObjectInspector instance stands for a specific type * and a specific way to store the data of that type in the memory. * - * For native java Object, we can directly access the internal structure through - * member fields and methods. ObjectInspector is a way to delegate that functionality - * away from the Object, so that we have more control on the behavior of those actions. + * For native java Object, we can directly access the internal structure through + * member fields and methods. ObjectInspector is a way to delegate that + * functionality away from the Object, so that we have more control on the + * behavior of those actions. * - * An efficient implementation of ObjectInspector should rely on factory, so that we can - * make sure the same ObjectInspector only has one instance. That also makes sure - * hashCode() and equals() methods of java.lang.Object directly works for ObjectInspector - * as well. + * An efficient implementation of ObjectInspector should rely on factory, so + * that we can make sure the same ObjectInspector only has one instance. That + * also makes sure hashCode() and equals() methods of java.lang.Object directly + * works for ObjectInspector as well. */ public interface ObjectInspector { @@ -41,22 +42,19 @@ }; /** - * Returns the name of the data type that is inspected by this ObjectInspector. - * This is used to display the type information to the user. + * Returns the name of the data type that is inspected by this + * ObjectInspector. This is used to display the type information to the user. * - * For primitive types, the type name is standardized. - * For other types, the type name can be something like "list", "map", - * java class names, or user-defined type names similar to typedef. + * For primitive types, the type name is standardized. For other types, the + * type name can be something like "list", "map", java class + * names, or user-defined type names similar to typedef. */ public String getTypeName(); - + /** - * An ObjectInspector must inherit from one of the following interfaces - * if getCategory() returns: - * PRIMITIVE: PrimitiveObjectInspector - * LIST: ListObjectInspector - * MAP: MapObjectInspector - * STRUCT: StructObjectInspector + * An ObjectInspector must inherit from one of the following interfaces if + * getCategory() returns: PRIMITIVE: PrimitiveObjectInspector LIST: + * ListObjectInspector MAP: MapObjectInspector STRUCT: StructObjectInspector */ public Category getCategory(); Index: serde/src/java/org/apache/hadoop/hive/serde2/ByteStream.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/ByteStream.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/ByteStream.java (working copy) @@ -18,38 +18,58 @@ package org.apache.hadoop.hive.serde2; - import org.apache.hadoop.hive.common.io.NonSyncByteArrayInputStream; import org.apache.hadoop.hive.common.io.NonSyncByteArrayOutputStream; /** * Extensions to bytearrayinput/output streams - * + * */ public class ByteStream { public static class Input extends NonSyncByteArrayInputStream { - public byte[] getData() { return buf; } - public int getCount() { return count;} - public void reset(byte [] argBuf, int argCount) { - buf = argBuf; mark = pos = 0; count = argCount; + public byte[] getData() { + return buf; } + + public int getCount() { + return count; + } + + public void reset(byte[] argBuf, int argCount) { + buf = argBuf; + mark = pos = 0; + count = argCount; + } + public Input() { - super(new byte [1]); + super(new byte[1]); } public Input(byte[] buf) { super(buf); } + public Input(byte[] buf, int offset, int length) { super(buf, offset, length); } } - + public static class Output extends NonSyncByteArrayOutputStream { - public byte[] getData() { return buf; } - public int getCount() { return count;} + @Override + public byte[] getData() { + return buf; + } - public Output() { super(); } - public Output(int size) { super(size); } + public int getCount() { + return count; + } + + public Output() { + super(); + } + + public Output(int size) { + super(size); + } } } Index: serde/src/java/org/apache/hadoop/hive/serde2/thrift/TReflectionUtils.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/thrift/TReflectionUtils.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/thrift/TReflectionUtils.java (working copy) @@ -20,25 +20,23 @@ import org.apache.thrift.protocol.TProtocolFactory; - public class TReflectionUtils { public static final String thriftReaderFname = "read"; public static final String thriftWriterFname = "write"; - public static final Class [] thriftRWParams; + public static final Class[] thriftRWParams; static { try { - thriftRWParams = new Class [] { - Class.forName("org.apache.thrift.protocol.TProtocol") - }; + thriftRWParams = new Class[] { Class + .forName("org.apache.thrift.protocol.TProtocol") }; } catch (ClassNotFoundException e) { throw new RuntimeException(e); } } public static TProtocolFactory getProtocolFactoryByName(String protocolName) - throws Exception { + throws Exception { Class protoClass = Class.forName(protocolName + "$Factory"); - return ((TProtocolFactory)protoClass.newInstance()); + return ((TProtocolFactory) protoClass.newInstance()); } } Index: serde/src/java/org/apache/hadoop/hive/serde2/thrift/TCTLSeparatedProtocol.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/thrift/TCTLSeparatedProtocol.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/thrift/TCTLSeparatedProtocol.java (working copy) @@ -16,42 +16,49 @@ * limitations under the License. */ - package org.apache.hadoop.hive.serde2.thrift; +import java.io.EOFException; +import java.nio.charset.CharacterCodingException; +import java.util.ArrayList; +import java.util.NoSuchElementException; +import java.util.Properties; +import java.util.StringTokenizer; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.serde.Constants; import org.apache.hadoop.io.Text; - import org.apache.thrift.TException; -import org.apache.thrift.transport.*; -import org.apache.thrift.*; -import org.apache.thrift.protocol.*; -import java.util.*; -import java.util.regex.Pattern; -import java.util.regex.Matcher; -import java.io.*; -import java.nio.ByteBuffer; -import java.nio.charset.CharacterCodingException; +import org.apache.thrift.protocol.TField; +import org.apache.thrift.protocol.TList; +import org.apache.thrift.protocol.TMap; +import org.apache.thrift.protocol.TMessage; +import org.apache.thrift.protocol.TProtocol; +import org.apache.thrift.protocol.TProtocolFactory; +import org.apache.thrift.protocol.TSet; +import org.apache.thrift.protocol.TStruct; +import org.apache.thrift.protocol.TType; +import org.apache.thrift.transport.TTransport; +import org.apache.thrift.transport.TTransportException; -import org.apache.hadoop.conf.Configuration; -import java.util.Properties; - /** - * - * An implementation of the Thrift Protocol for ctl separated - * records. - * This is not thrift compliant in that it doesn't write out field ids - * so things cannot actually be versioned. + * + * An implementation of the Thrift Protocol for ctl separated records. This is + * not thrift compliant in that it doesn't write out field ids so things cannot + * actually be versioned. */ -public class TCTLSeparatedProtocol extends TProtocol - implements ConfigurableTProtocol, WriteNullsProtocol, SkippableTProtocol { +public class TCTLSeparatedProtocol extends TProtocol implements + ConfigurableTProtocol, WriteNullsProtocol, SkippableTProtocol { - final static Log LOG = LogFactory.getLog(TCTLSeparatedProtocol.class.getName()); + final static Log LOG = LogFactory.getLog(TCTLSeparatedProtocol.class + .getName()); - static byte ORDERED_TYPE = (byte)-1; - + static byte ORDERED_TYPE = (byte) -1; + /** * Factory for JSON protocol objects */ @@ -83,21 +90,31 @@ protected Pattern mapPattern; /** - * The quote character when supporting quotes with ability to not split across quoted entries. Like csv. - * Note that escaping the quote is not currently supported. + * The quote character when supporting quotes with ability to not split across + * quoted entries. Like csv. Note that escaping the quote is not currently + * supported. */ protected String quote; - /** * Inspect the separators this instance is configured with. */ - public String getPrimarySeparator() { return primarySeparator; } - public String getSecondarySeparator() { return secondarySeparator; } - public String getRowSeparator() { return rowSeparator; } - public String getMapSeparator() { return mapSeparator; } + public String getPrimarySeparator() { + return primarySeparator; + } + public String getSecondarySeparator() { + return secondarySeparator; + } + public String getRowSeparator() { + return rowSeparator; + } + + public String getMapSeparator() { + return mapSeparator; + } + /** * The transport stream is tokenized on the row separator */ @@ -124,7 +141,6 @@ */ protected int innerIndex; - /** * Is this the first field we're writing */ @@ -135,41 +151,38 @@ */ protected boolean firstInnerField; - /** * Are we writing a map and need to worry about k/v separator? */ protected boolean isMap; - /** - * For writes, on what element are we on so we know when to use normal list separator or - * for a map know when to use the k/v separator + * For writes, on what element are we on so we know when to use normal list + * separator or for a map know when to use the k/v separator */ protected long elemIndex; - /** * Are we currently on the top-level columns or parsing a column itself */ protected boolean inner; - /** - * For places where the separators are back to back, should we return a null or an empty string since it is ambiguous. - * This also applies to extra columns that are read but aren't in the current record. + * For places where the separators are back to back, should we return a null + * or an empty string since it is ambiguous. This also applies to extra + * columns that are read but aren't in the current record. */ protected boolean returnNulls; /** * The transport being wrapped. - * + * */ final protected TTransport innerTransport; - /** - * Strings used to lookup the various configurable paramaters of this protocol. + * Strings used to lookup the various configurable paramaters of this + * protocol. */ public final static String ReturnNullsKey = "separators.return_nulls"; public final static String BufferSizeKey = "separators.buffer_size"; @@ -186,10 +199,9 @@ /** * The nullString in UTF-8 bytes - */ + */ protected Text nullText; - /** * A convenience class for tokenizing a TTransport */ @@ -201,63 +213,65 @@ final String separator; byte buf[]; - public SimpleTransportTokenizer(TTransport trans, String separator, int buffer_length) { + public SimpleTransportTokenizer(TTransport trans, String separator, + int buffer_length) { this.trans = trans; this.separator = separator; buf = new byte[buffer_length]; - // do not fill tokenizer until user requests since filling it could read in data + // do not fill tokenizer until user requests since filling it could read + // in data // not meant for this instantiation. fillTokenizer(); } private boolean fillTokenizer() { try { - int length = trans.read(buf, 0, buf.length); - if(length <=0 ) { - tokenizer = new StringTokenizer("", separator, true); - return false; - } - String row; - try { - row = Text.decode(buf, 0, length); - } catch (CharacterCodingException e) { - throw new RuntimeException(e); - } - tokenizer = new StringTokenizer(row, separator, true); - } catch(TTransportException e) { - e.printStackTrace(); - tokenizer = null; + int length = trans.read(buf, 0, buf.length); + if (length <= 0) { + tokenizer = new StringTokenizer("", separator, true); return false; } - return true; + String row; + try { + row = Text.decode(buf, 0, length); + } catch (CharacterCodingException e) { + throw new RuntimeException(e); + } + tokenizer = new StringTokenizer(row, separator, true); + } catch (TTransportException e) { + e.printStackTrace(); + tokenizer = null; + return false; + } + return true; } public String nextToken() throws EOFException { StringBuffer ret = null; boolean done = false; - if(tokenizer == null) { + if (tokenizer == null) { fillTokenizer(); } - while(! done) { + while (!done) { - if(! tokenizer.hasMoreTokens()) { - if(! fillTokenizer()) { + if (!tokenizer.hasMoreTokens()) { + if (!fillTokenizer()) { break; } } try { final String nextToken = tokenizer.nextToken(); - if(nextToken.equals(separator)) { + if (nextToken.equals(separator)) { done = true; - } else if(ret == null) { + } else if (ret == null) { ret = new StringBuffer(nextToken); } else { ret.append(nextToken); } - } catch(NoSuchElementException e) { + } catch (NoSuchElementException e) { if (ret == null) { throw new EOFException(e.getMessage()); } @@ -269,34 +283,45 @@ } }; - /** - * The simple constructor which assumes ctl-a, ctl-b and '\n' separators and to return empty strings for empty fields. - * - * @param trans - the ttransport to use as input or output - * + * The simple constructor which assumes ctl-a, ctl-b and '\n' separators and + * to return empty strings for empty fields. + * + * @param trans + * - the ttransport to use as input or output + * */ public TCTLSeparatedProtocol(TTransport trans) { - this(trans, defaultPrimarySeparator, defaultSecondarySeparator, defaultMapSeparator, defaultRowSeparator, true, 4096); + this(trans, defaultPrimarySeparator, defaultSecondarySeparator, + defaultMapSeparator, defaultRowSeparator, true, 4096); } public TCTLSeparatedProtocol(TTransport trans, int buffer_size) { - this(trans, defaultPrimarySeparator, defaultSecondarySeparator, defaultMapSeparator, defaultRowSeparator, true, buffer_size); + this(trans, defaultPrimarySeparator, defaultSecondarySeparator, + defaultMapSeparator, defaultRowSeparator, true, buffer_size); } /** - * @param trans - the ttransport to use as input or output - * @param primarySeparator the separator between columns (aka fields) - * @param secondarySeparator the separator within a field for things like sets and maps and lists - * @param mapSeparator - the key/value separator - * @param rowSeparator - the record separator - * @param returnNulls - whether to return a null or an empty string for fields that seem empty (ie two primary separators back to back) + * @param trans + * - the ttransport to use as input or output + * @param primarySeparator + * the separator between columns (aka fields) + * @param secondarySeparator + * the separator within a field for things like sets and maps and + * lists + * @param mapSeparator + * - the key/value separator + * @param rowSeparator + * - the record separator + * @param returnNulls + * - whether to return a null or an empty string for fields that seem + * empty (ie two primary separators back to back) */ - public TCTLSeparatedProtocol(TTransport trans, String primarySeparator, String secondarySeparator, String mapSeparator, String rowSeparator, - boolean returnNulls, - int bufferSize) { + public TCTLSeparatedProtocol(TTransport trans, String primarySeparator, + String secondarySeparator, String mapSeparator, String rowSeparator, + boolean returnNulls, int bufferSize) { super(trans); this.returnNulls = returnNulls; @@ -306,51 +331,52 @@ this.rowSeparator = rowSeparator; this.mapSeparator = mapSeparator; - this.innerTransport = trans; + innerTransport = trans; this.bufferSize = bufferSize; - this.nullString = "\\N"; + nullString = "\\N"; } - - /** * Sets the internal separator patterns and creates the internal tokenizer. */ protected void internalInitialize() { - // in the future could allow users to specify a quote character that doesn't need escaping but for now ... - final String primaryPatternString = - quote == null ? primarySeparator : - "(?:^|" + primarySeparator + ")(" + quote + "(?:[^" + quote + "]+|" + quote + quote + ")*" + quote + "|[^" + primarySeparator + "]*)"; + // in the future could allow users to specify a quote character that doesn't + // need escaping but for now ... + final String primaryPatternString = quote == null ? primarySeparator + : "(?:^|" + primarySeparator + ")(" + quote + "(?:[^" + quote + "]+|" + + quote + quote + ")*" + quote + "|[^" + primarySeparator + "]*)"; if (quote != null) { stripSeparatorPrefix = Pattern.compile("^" + primarySeparator); stripQuotePrefix = Pattern.compile("^" + quote); stripQuotePostfix = Pattern.compile(quote + "$"); - } + } primaryPattern = Pattern.compile(primaryPatternString); secondaryPattern = Pattern.compile(secondarySeparator); mapPattern = Pattern.compile(secondarySeparator + "|" + mapSeparator); nullText = new Text(nullString); - transportTokenizer = new SimpleTransportTokenizer(innerTransport, rowSeparator, bufferSize); + transportTokenizer = new SimpleTransportTokenizer(innerTransport, + rowSeparator, bufferSize); } /** - * For quoted fields, strip away the quotes and also need something to strip away the control separator when using - * complex split method defined here. + * For quoted fields, strip away the quotes and also need something to strip + * away the control separator when using complex split method defined here. */ protected Pattern stripSeparatorPrefix; protected Pattern stripQuotePrefix; protected Pattern stripQuotePostfix; - - /** - * + /** + * * Split the line based on a complex regex pattern - * - * @param line the current row - * @param p the pattern for matching fields in the row + * + * @param line + * the current row + * @param p + * the pattern for matching fields in the row * @return List of Strings - not including the separator in them */ protected String[] complexSplit(String line, Pattern p) { @@ -360,104 +386,116 @@ // For each field while (m.find()) { String match = m.group(); - if (match == null) + if (match == null) { break; - if (match.length() == 0) + } + if (match.length() == 0) { match = null; - else { - if(stripSeparatorPrefix.matcher(match).find()) { + } else { + if (stripSeparatorPrefix.matcher(match).find()) { match = match.substring(1); } - if(stripQuotePrefix.matcher(match).find()) { + if (stripQuotePrefix.matcher(match).find()) { match = match.substring(1); } - if(stripQuotePostfix.matcher(match).find()) { - match = match.substring(0,match.length() - 1); + if (stripQuotePostfix.matcher(match).find()) { + match = match.substring(0, match.length() - 1); } } list.add(match); } - return (String[])list.toArray(new String[1]); + return list.toArray(new String[1]); } - - protected String getByteValue(String altValue, String defaultVal) { if (altValue != null && altValue.length() > 0) { try { - byte b [] = new byte[1]; + byte b[] = new byte[1]; b[0] = Byte.valueOf(altValue).byteValue(); return new String(b); - } catch(NumberFormatException e) { + } catch (NumberFormatException e) { return altValue; } } return defaultVal; } - /** * Initialize the TProtocol - * @param conf System properties - * @param tbl table properties + * + * @param conf + * System properties + * @param tbl + * table properties * @throws TException */ public void initialize(Configuration conf, Properties tbl) throws TException { + primarySeparator = getByteValue(tbl.getProperty(Constants.FIELD_DELIM), + primarySeparator); + secondarySeparator = getByteValue(tbl + .getProperty(Constants.COLLECTION_DELIM), secondarySeparator); + rowSeparator = getByteValue(tbl.getProperty(Constants.LINE_DELIM), + rowSeparator); + mapSeparator = getByteValue(tbl.getProperty(Constants.MAPKEY_DELIM), + mapSeparator); + returnNulls = Boolean.valueOf( + tbl.getProperty(ReturnNullsKey, String.valueOf(returnNulls))) + .booleanValue(); + bufferSize = Integer.valueOf( + tbl.getProperty(BufferSizeKey, String.valueOf(bufferSize))).intValue(); + nullString = tbl.getProperty(Constants.SERIALIZATION_NULL_FORMAT, "\\N"); + quote = tbl.getProperty(Constants.QUOTE_CHAR, null); - primarySeparator = getByteValue(tbl.getProperty(Constants.FIELD_DELIM), primarySeparator); - secondarySeparator = getByteValue(tbl.getProperty(Constants.COLLECTION_DELIM), secondarySeparator); - rowSeparator = getByteValue(tbl.getProperty(Constants.LINE_DELIM), rowSeparator); - mapSeparator = getByteValue(tbl.getProperty(Constants.MAPKEY_DELIM), mapSeparator); - returnNulls = Boolean.valueOf(tbl.getProperty(ReturnNullsKey, String.valueOf(returnNulls))).booleanValue(); - bufferSize = Integer.valueOf(tbl.getProperty(BufferSizeKey, String.valueOf(bufferSize))).intValue(); - nullString = tbl.getProperty(Constants.SERIALIZATION_NULL_FORMAT, "\\N"); - quote = tbl.getProperty(Constants.QUOTE_CHAR, null); - internalInitialize(); } + @Override public void writeMessageBegin(TMessage message) throws TException { } + @Override public void writeMessageEnd() throws TException { } + @Override public void writeStructBegin(TStruct struct) throws TException { firstField = true; } + @Override public void writeStructEnd() throws TException { - // We don't write rowSeparatorByte because that should be handled by file format. + // We don't write rowSeparatorByte because that should be handled by file + // format. } + @Override public void writeFieldBegin(TField field) throws TException { - if(! firstField) { + if (!firstField) { internalWriteString(primarySeparator); } firstField = false; } + @Override public void writeFieldEnd() throws TException { } + @Override public void writeFieldStop() { } + @Override public void writeMapBegin(TMap map) throws TException { // nesting not allowed! - if(map.keyType == TType.STRUCT || - map.keyType == TType.MAP || - map.keyType == TType.LIST || - map.keyType == TType.SET) { + if (map.keyType == TType.STRUCT || map.keyType == TType.MAP + || map.keyType == TType.LIST || map.keyType == TType.SET) { throw new TException("Not implemented: nested structures"); } // nesting not allowed! - if(map.valueType == TType.STRUCT || - map.valueType == TType.MAP || - map.valueType == TType.LIST || - map.valueType == TType.SET) { + if (map.valueType == TType.STRUCT || map.valueType == TType.MAP + || map.valueType == TType.LIST || map.valueType == TType.SET) { throw new TException("Not implemented: nested structures"); } @@ -467,71 +505,80 @@ elemIndex = 0; } + @Override public void writeMapEnd() throws TException { isMap = false; inner = false; } + @Override public void writeListBegin(TList list) throws TException { - if(list.elemType == TType.STRUCT || - list.elemType == TType.MAP || - list.elemType == TType.LIST || - list.elemType == TType.SET) { + if (list.elemType == TType.STRUCT || list.elemType == TType.MAP + || list.elemType == TType.LIST || list.elemType == TType.SET) { throw new TException("Not implemented: nested structures"); } firstInnerField = true; inner = true; } + @Override public void writeListEnd() throws TException { inner = false; } - + + @Override public void writeSetBegin(TSet set) throws TException { - if(set.elemType == TType.STRUCT || - set.elemType == TType.MAP || - set.elemType == TType.LIST || - set.elemType == TType.SET) { + if (set.elemType == TType.STRUCT || set.elemType == TType.MAP + || set.elemType == TType.LIST || set.elemType == TType.SET) { throw new TException("Not implemented: nested structures"); } firstInnerField = true; inner = true; } + @Override public void writeSetEnd() throws TException { inner = false; } + @Override public void writeBool(boolean b) throws TException { writeString(String.valueOf(b)); } // for writing out single byte - private byte buf[] = new byte[1]; + private final byte buf[] = new byte[1]; + + @Override public void writeByte(byte b) throws TException { buf[0] = b; trans_.write(buf); } + @Override public void writeI16(short i16) throws TException { writeString(String.valueOf(i16)); } + @Override public void writeI32(int i32) throws TException { writeString(String.valueOf(i32)); } + @Override public void writeI64(long i64) throws TException { writeString(String.valueOf(i64)); } + @Override public void writeDouble(double dub) throws TException { writeString(String.valueOf(dub)); } Text tmpText = new Text(); - public void internalWriteString(String str) throws TException { - if(str != null) { + + public void internalWriteString(String str) throws TException { + if (str != null) { tmpText.set(str); trans_.write(tmpText.getBytes(), 0, tmpText.getLength()); } else { @@ -539,11 +586,13 @@ } } + @Override public void writeString(String str) throws TException { - if(inner) { - if(!firstInnerField) { - // super hack city notice the mod plus only happens after firstfield hit, so == 0 is right. - if(isMap && elemIndex++ % 2 == 0) { + if (inner) { + if (!firstInnerField) { + // super hack city notice the mod plus only happens after firstfield + // hit, so == 0 is right. + if (isMap && elemIndex++ % 2 == 0) { internalWriteString(mapSeparator); } else { internalWriteString(secondarySeparator); @@ -555,72 +604,79 @@ internalWriteString(str); } + @Override public void writeBinary(byte[] bin) throws TException { - throw new TException("Ctl separated protocol cannot support writing Binary data!"); + throw new TException( + "Ctl separated protocol cannot support writing Binary data!"); } + @Override public TMessage readMessageBegin() throws TException { - return new TMessage(); + return new TMessage(); } + @Override public void readMessageEnd() throws TException { } - public TStruct readStructBegin() throws TException { - assert(!inner); - try { - final String tmp = transportTokenizer.nextToken(); - columns = quote == null ? primaryPattern.split(tmp) : complexSplit(tmp, primaryPattern); - index = 0; - return new TStruct(); - } catch(EOFException e) { - return null; - } + @Override + public TStruct readStructBegin() throws TException { + assert (!inner); + try { + final String tmp = transportTokenizer.nextToken(); + columns = quote == null ? primaryPattern.split(tmp) : complexSplit(tmp, + primaryPattern); + index = 0; + return new TStruct(); + } catch (EOFException e) { + return null; + } } + @Override public void readStructEnd() throws TException { columns = null; } - /** - * Skip past the current field - * Just increments the field index counter. + * Skip past the current field Just increments the field index counter. */ public void skip(byte type) { - if( inner) { + if (inner) { innerIndex++; } else { index++; } } - + @Override public TField readFieldBegin() throws TException { - assert( !inner); - TField f = new TField("", ORDERED_TYPE, (short)-1); - // slight hack to communicate to DynamicSerDe that the field ids are not being set but things are ordered. - return f; + assert (!inner); + TField f = new TField("", ORDERED_TYPE, (short) -1); + // slight hack to communicate to DynamicSerDe that the field ids are not + // being set but things are ordered. + return f; } + @Override public void readFieldEnd() throws TException { fields = null; } + @Override public TMap readMapBegin() throws TException { - assert( !inner); + assert (!inner); TMap map = new TMap(); - if(columns[index] == null || - columns[index].equals(nullString)) { + if (columns[index] == null || columns[index].equals(nullString)) { index++; - if(returnNulls) { + if (returnNulls) { return null; } - } else if(columns[index].isEmpty()) { + } else if (columns[index].isEmpty()) { index++; } else { fields = mapPattern.split(columns[index++]); - map = new TMap(ORDERED_TYPE, ORDERED_TYPE, fields.length/2); + map = new TMap(ORDERED_TYPE, ORDERED_TYPE, fields.length / 2); } innerIndex = 0; inner = true; @@ -628,21 +684,22 @@ return map; } + @Override public void readMapEnd() throws TException { inner = false; isMap = false; } + @Override public TList readListBegin() throws TException { - assert( !inner); + assert (!inner); TList list = new TList(); - if(columns[index] == null || - columns[index].equals(nullString)) { + if (columns[index] == null || columns[index].equals(nullString)) { index++; - if(returnNulls) { + if (returnNulls) { return null; } - } else if(columns[index].isEmpty()) { + } else if (columns[index].isEmpty()) { index++; } else { fields = secondaryPattern.split(columns[index++]); @@ -653,20 +710,21 @@ return list; } + @Override public void readListEnd() throws TException { inner = false; } + @Override public TSet readSetBegin() throws TException { - assert( !inner); + assert (!inner); TSet set = new TSet(); - if(columns[index] == null || - columns[index].equals(nullString)) { + if (columns[index] == null || columns[index].equals(nullString)) { index++; - if(returnNulls) { + if (returnNulls) { return null; } - } else if(columns[index].isEmpty()) { + } else if (columns[index].isEmpty()) { index++; } else { fields = secondaryPattern.split(columns[index++]); @@ -687,16 +745,20 @@ writeString(null); } + @Override public void readSetEnd() throws TException { inner = false; } + @Override public boolean readBool() throws TException { String val = readString(); lastPrimitiveWasNullFlag = val == null; - return val == null || val.isEmpty() ? false : Boolean.valueOf(val).booleanValue(); + return val == null || val.isEmpty() ? false : Boolean.valueOf(val) + .booleanValue(); } + @Override public byte readByte() throws TException { String val = readString(); lastPrimitiveWasNullFlag = val == null; @@ -708,6 +770,7 @@ } } + @Override public short readI16() throws TException { String val = readString(); lastPrimitiveWasNullFlag = val == null; @@ -719,6 +782,7 @@ } } + @Override public int readI32() throws TException { String val = readString(); lastPrimitiveWasNullFlag = val == null; @@ -730,6 +794,7 @@ } } + @Override public long readI64() throws TException { String val = readString(); lastPrimitiveWasNullFlag = val == null; @@ -741,32 +806,38 @@ } } + @Override public double readDouble() throws TException { String val = readString(); lastPrimitiveWasNullFlag = val == null; try { - return val == null || val.isEmpty() ? 0 :Double.valueOf(val).doubleValue(); + return val == null || val.isEmpty() ? 0 : Double.valueOf(val) + .doubleValue(); } catch (NumberFormatException e) { lastPrimitiveWasNullFlag = true; return 0; } } + @Override public String readString() throws TException { String ret; - if(!inner) { - ret = columns != null && index < columns.length ? columns[index] : null; + if (!inner) { + ret = columns != null && index < columns.length ? columns[index] : null; index++; } else { - ret = fields != null && innerIndex < fields.length ? fields[innerIndex] : null; + ret = fields != null && innerIndex < fields.length ? fields[innerIndex] + : null; innerIndex++; } - if(ret == null || ret.equals(nullString)) + if (ret == null || ret.equals(nullString)) { return returnNulls ? null : ""; - else + } else { return ret; + } } + @Override public byte[] readBinary() throws TException { throw new TException("Not implemented for control separated data"); } Index: serde/src/java/org/apache/hadoop/hive/serde2/thrift/WriteTextProtocol.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/thrift/WriteTextProtocol.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/thrift/WriteTextProtocol.java (working copy) @@ -19,13 +19,12 @@ package org.apache.hadoop.hive.serde2.thrift; import org.apache.hadoop.io.Text; - import org.apache.thrift.TException; /** * An interface for TProtocols that can write out data in hadoop Text objects - * (UTF-8 encoded String). This helps a lot with performance because we don't - * need to do unnecessary UTF-8 decoding and encoding loops. + * (UTF-8 encoded String). This helps a lot with performance because we don't + * need to do unnecessary UTF-8 decoding and encoding loops. */ public interface WriteTextProtocol { Index: serde/src/java/org/apache/hadoop/hive/serde2/thrift/ThriftDeserializer.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/thrift/ThriftDeserializer.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/thrift/ThriftDeserializer.java (working copy) @@ -31,27 +31,33 @@ private ThriftByteStreamTypedSerDe tsd; - public ThriftDeserializer() { } - - public void initialize(Configuration job, Properties tbl) throws SerDeException { + public ThriftDeserializer() { + } + + public void initialize(Configuration job, Properties tbl) + throws SerDeException { try { // both the classname and the protocol name are Table properties // the only hardwired assumption is that records are fixed on a // per Table basis - String className = tbl.getProperty(org.apache.hadoop.hive.serde.Constants.SERIALIZATION_CLASS); + String className = tbl + .getProperty(org.apache.hadoop.hive.serde.Constants.SERIALIZATION_CLASS); Class recordClass = job.getClassByName(className); - String protoName = tbl.getProperty(org.apache.hadoop.hive.serde.Constants.SERIALIZATION_FORMAT); + String protoName = tbl + .getProperty(org.apache.hadoop.hive.serde.Constants.SERIALIZATION_FORMAT); if (protoName == null) { protoName = "TBinaryProtocol"; } // For backward compatibility - protoName = protoName.replace("com.facebook.thrift.protocol", "org.apache.thrift.protocol"); + protoName = protoName.replace("com.facebook.thrift.protocol", + "org.apache.thrift.protocol"); - TProtocolFactory tp = TReflectionUtils.getProtocolFactoryByName(protoName); + TProtocolFactory tp = TReflectionUtils + .getProtocolFactoryByName(protoName); tsd = new ThriftByteStreamTypedSerDe(recordClass, tp, tp); - + } catch (Exception e) { throw new SerDeException(e); } @@ -60,7 +66,7 @@ public Object deserialize(Writable field) throws SerDeException { return tsd.deserialize(field); } - + public ObjectInspector getObjectInspector() throws SerDeException { return tsd.getObjectInspector(); } Index: serde/src/java/org/apache/hadoop/hive/serde2/thrift/ConfigurableTProtocol.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/thrift/ConfigurableTProtocol.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/thrift/ConfigurableTProtocol.java (working copy) @@ -18,21 +18,24 @@ package org.apache.hadoop.hive.serde2.thrift; +import java.util.Properties; + import org.apache.hadoop.conf.Configuration; -import java.util.Properties; import org.apache.thrift.TException; /** - * An interface for TProtocols that need to have properties passed in to - * initialize them. e.g., separators for TCTLSeparatedProtocol. - * If there was a regex like deserializer, the regex could be passed in - * in this manner. + * An interface for TProtocols that need to have properties passed in to + * initialize them. e.g., separators for TCTLSeparatedProtocol. If there was a + * regex like deserializer, the regex could be passed in in this manner. */ public interface ConfigurableTProtocol { /** * Initialize the TProtocol - * @param conf System properties - * @param tbl table properties + * + * @param conf + * System properties + * @param tbl + * table properties * @throws TException */ public void initialize(Configuration conf, Properties tbl) throws TException; Index: serde/src/java/org/apache/hadoop/hive/serde2/thrift/TBinarySortableProtocol.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/thrift/TBinarySortableProtocol.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/thrift/TBinarySortableProtocol.java (working copy) @@ -16,61 +16,65 @@ * limitations under the License. */ - package org.apache.hadoop.hive.serde2.thrift; +import java.io.UnsupportedEncodingException; +import java.util.Arrays; +import java.util.Properties; + import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; - -import org.apache.thrift.TException; -import org.apache.thrift.transport.*; -import org.apache.thrift.protocol.*; -import java.util.*; -import java.io.*; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.serde.Constants; import org.apache.hadoop.io.Text; +import org.apache.thrift.TException; +import org.apache.thrift.protocol.TField; +import org.apache.thrift.protocol.TList; +import org.apache.thrift.protocol.TMap; +import org.apache.thrift.protocol.TMessage; +import org.apache.thrift.protocol.TProtocol; +import org.apache.thrift.protocol.TProtocolFactory; +import org.apache.thrift.protocol.TSet; +import org.apache.thrift.protocol.TStruct; +import org.apache.thrift.transport.TTransport; -import java.util.Properties; - /** * An implementation of the Thrift Protocol for binary sortable records. * - * The data format: - * NULL: a single byte \0 - * NON-NULL Primitives: ALWAYS prepend a single byte \1, and then: - * Boolean: FALSE = \1, TRUE = \2 - * Byte: flip the sign-bit to make sure negative comes before positive - * Short: flip the sign-bit to make sure negative comes before positive - * Int: flip the sign-bit to make sure negative comes before positive - * Long: flip the sign-bit to make sure negative comes before positive - * Double: flip the sign-bit for positive double, and all bits for negative double values - * String: NULL-terminated UTF-8 string, with NULL escaped to \1 \1, and \1 escaped to \1 \2 - * NON-NULL Complex Types: - * Struct: first the single byte \1, and then one field by one field. - * List: size stored as Int (see above), then one element by one element. - * Map: size stored as Int (see above), then one key by one value, and then the next pair and so on. - * Binary: size stored as Int (see above), then the binary data in its original form - * - * Note that the relative order of list/map/binary will be based on the size first (and elements one by one if - * the sizes are equal). + * The data format: NULL: a single byte \0 NON-NULL Primitives: ALWAYS prepend a + * single byte \1, and then: Boolean: FALSE = \1, TRUE = \2 Byte: flip the + * sign-bit to make sure negative comes before positive Short: flip the sign-bit + * to make sure negative comes before positive Int: flip the sign-bit to make + * sure negative comes before positive Long: flip the sign-bit to make sure + * negative comes before positive Double: flip the sign-bit for positive double, + * and all bits for negative double values String: NULL-terminated UTF-8 string, + * with NULL escaped to \1 \1, and \1 escaped to \1 \2 NON-NULL Complex Types: + * Struct: first the single byte \1, and then one field by one field. List: size + * stored as Int (see above), then one element by one element. Map: size stored + * as Int (see above), then one key by one value, and then the next pair and so + * on. Binary: size stored as Int (see above), then the binary data in its + * original form * - * This protocol takes an additional parameter SERIALIZATION_SORT_ORDER which is a string containing only "+" and "-". - * The length of the string should equal to the number of fields in the top-level struct for serialization. - * "+" means the field should be sorted ascendingly, and "-" means descendingly. The sub fields in the same top-level - * field will have the same sort order. + * Note that the relative order of list/map/binary will be based on the size + * first (and elements one by one if the sizes are equal). * - * This is not thrift compliant in that it doesn't write out field ids - * so things cannot actually be versioned. + * This protocol takes an additional parameter SERIALIZATION_SORT_ORDER which is + * a string containing only "+" and "-". The length of the string should equal + * to the number of fields in the top-level struct for serialization. "+" means + * the field should be sorted ascendingly, and "-" means descendingly. The sub + * fields in the same top-level field will have the same sort order. + * + * This is not thrift compliant in that it doesn't write out field ids so things + * cannot actually be versioned. */ -public class TBinarySortableProtocol extends TProtocol implements ConfigurableTProtocol, - WriteNullsProtocol, WriteTextProtocol { +public class TBinarySortableProtocol extends TProtocol implements + ConfigurableTProtocol, WriteNullsProtocol, WriteTextProtocol { - final static Log LOG = LogFactory.getLog(TBinarySortableProtocol.class.getName()); + final static Log LOG = LogFactory.getLog(TBinarySortableProtocol.class + .getName()); - static byte ORDERED_TYPE = (byte)-1; - + static byte ORDERED_TYPE = (byte) -1; + /** * Factory for TBinarySortableProtocol objects */ @@ -87,75 +91,89 @@ } /** - * The stack level of the current field. Top-level fields have a stackLevel value of 1. - * Each nested struct/list/map will increase the stackLevel value by 1. + * The stack level of the current field. Top-level fields have a stackLevel + * value of 1. Each nested struct/list/map will increase the stackLevel value + * by 1. */ int stackLevel; /** - * The field ID in the top level struct. This is used to determine whether this field - * should be sorted ascendingly or descendingly. + * The field ID in the top level struct. This is used to determine whether + * this field should be sorted ascendingly or descendingly. */ int topLevelStructFieldID; /** - * A string that consists of only "+" and "-". It should have the same length as the number - * of fields in the top level struct. "+" means the corresponding field is sorted ascendingly - * and "-" means the corresponding field is sorted descendingly. + * A string that consists of only "+" and "-". It should have the same length + * as the number of fields in the top level struct. "+" means the + * corresponding field is sorted ascendingly and "-" means the corresponding + * field is sorted descendingly. */ String sortOrder; /** - * Whether the current field is sorted ascendingly. Always equals to + * Whether the current field is sorted ascendingly. Always equals to * sortOrder.charAt(topLevelStructFieldID) != '-' */ - boolean ascending; - + boolean ascending; + public void initialize(Configuration conf, Properties tbl) throws TException { sortOrder = tbl.getProperty(Constants.SERIALIZATION_SORT_ORDER); if (sortOrder == null) { sortOrder = ""; } - for(int i=0; i= sortOrder.length() || sortOrder.charAt(topLevelStructFieldID) != '-'); + ascending = (topLevelStructFieldID >= sortOrder.length() || sortOrder + .charAt(topLevelStructFieldID) != '-'); } else { writeRawBytes(nonNullByte, 0, 1); - // If the struct is null and level > 1, DynamicSerDe will call writeNull(); - } + // If the struct is null and level > 1, DynamicSerDe will call + // writeNull(); + } } + @Override public void writeStructEnd() throws TException { - stackLevel --; + stackLevel--; } + @Override public void writeFieldBegin(TField field) throws TException { } + @Override public void writeFieldEnd() throws TException { if (stackLevel == 1) { - topLevelStructFieldID ++; - ascending = (topLevelStructFieldID >= sortOrder.length() || sortOrder.charAt(topLevelStructFieldID) != '-'); + topLevelStructFieldID++; + ascending = (topLevelStructFieldID >= sortOrder.length() || sortOrder + .charAt(topLevelStructFieldID) != '-'); } } + @Override public void writeFieldStop() { } + @Override public void writeMapBegin(TMap map) throws TException { stackLevel++; if (map == null) { @@ -165,10 +183,12 @@ } } + @Override public void writeMapEnd() throws TException { - stackLevel --; + stackLevel--; } + @Override public void writeListBegin(TList list) throws TException { stackLevel++; if (list == null) { @@ -178,10 +198,12 @@ } } + @Override public void writeListEnd() throws TException { - stackLevel --; + stackLevel--; } - + + @Override public void writeSetBegin(TSet set) throws TException { stackLevel++; if (set == null) { @@ -191,14 +213,17 @@ } } + @Override public void writeSetEnd() throws TException { - stackLevel --; + stackLevel--; } byte[] rawBytesBuffer; + // This method takes care of bit-flipping for descending order // Declare this method as final for performance reasons - final private void writeRawBytes(byte[] bytes, int begin, int length) throws TException { + final private void writeRawBytes(byte[] bytes, int begin, int length) + throws TException { if (ascending) { trans_.write(bytes, begin, length); } else { @@ -206,97 +231,109 @@ if (rawBytesBuffer == null || rawBytesBuffer.length < bytes.length) { rawBytesBuffer = new byte[bytes.length]; } - for(int i=begin; i> 8) ^ 0x80)); - i16out[1] = (byte)(0xff & (i16)); + i16out[0] = (byte) (0xff & ((i16 >> 8) ^ 0x80)); + i16out[1] = (byte) (0xff & (i16)); writeRawBytes(nonNullByte, 0, 1); writeRawBytes(i16out, 0, 2); } - private byte[] i32out = new byte[4]; + private final byte[] i32out = new byte[4]; + + @Override public void writeI32(int i32) throws TException { - i32out[0] = (byte)(0xff & ((i32 >> 24) ^ 0x80)); - i32out[1] = (byte)(0xff & (i32 >> 16)); - i32out[2] = (byte)(0xff & (i32 >> 8)); - i32out[3] = (byte)(0xff & (i32)); + i32out[0] = (byte) (0xff & ((i32 >> 24) ^ 0x80)); + i32out[1] = (byte) (0xff & (i32 >> 16)); + i32out[2] = (byte) (0xff & (i32 >> 8)); + i32out[3] = (byte) (0xff & (i32)); writeRawBytes(nonNullByte, 0, 1); writeRawBytes(i32out, 0, 4); } - private byte[] i64out = new byte[8]; + private final byte[] i64out = new byte[8]; + + @Override public void writeI64(long i64) throws TException { - i64out[0] = (byte)(0xff & ((i64 >> 56) ^ 0x80)); - i64out[1] = (byte)(0xff & (i64 >> 48)); - i64out[2] = (byte)(0xff & (i64 >> 40)); - i64out[3] = (byte)(0xff & (i64 >> 32)); - i64out[4] = (byte)(0xff & (i64 >> 24)); - i64out[5] = (byte)(0xff & (i64 >> 16)); - i64out[6] = (byte)(0xff & (i64 >> 8)); - i64out[7] = (byte)(0xff & (i64)); + i64out[0] = (byte) (0xff & ((i64 >> 56) ^ 0x80)); + i64out[1] = (byte) (0xff & (i64 >> 48)); + i64out[2] = (byte) (0xff & (i64 >> 40)); + i64out[3] = (byte) (0xff & (i64 >> 32)); + i64out[4] = (byte) (0xff & (i64 >> 24)); + i64out[5] = (byte) (0xff & (i64 >> 16)); + i64out[6] = (byte) (0xff & (i64 >> 8)); + i64out[7] = (byte) (0xff & (i64)); writeRawBytes(nonNullByte, 0, 1); writeRawBytes(i64out, 0, 8); } + @Override public void writeDouble(double dub) throws TException { long i64 = Double.doubleToLongBits(dub); if ((i64 & (1L << 63)) != 0) { // negative numbers, flip all bits - i64out[0] = (byte)(0xff & ((i64 >> 56) ^ 0xff)); - i64out[1] = (byte)(0xff & ((i64 >> 48) ^ 0xff)); - i64out[2] = (byte)(0xff & ((i64 >> 40) ^ 0xff)); - i64out[3] = (byte)(0xff & ((i64 >> 32) ^ 0xff)); - i64out[4] = (byte)(0xff & ((i64 >> 24) ^ 0xff)); - i64out[5] = (byte)(0xff & ((i64 >> 16) ^ 0xff)); - i64out[6] = (byte)(0xff & ((i64 >> 8) ^ 0xff)); - i64out[7] = (byte)(0xff & ((i64) ^ 0xff)); + i64out[0] = (byte) (0xff & ((i64 >> 56) ^ 0xff)); + i64out[1] = (byte) (0xff & ((i64 >> 48) ^ 0xff)); + i64out[2] = (byte) (0xff & ((i64 >> 40) ^ 0xff)); + i64out[3] = (byte) (0xff & ((i64 >> 32) ^ 0xff)); + i64out[4] = (byte) (0xff & ((i64 >> 24) ^ 0xff)); + i64out[5] = (byte) (0xff & ((i64 >> 16) ^ 0xff)); + i64out[6] = (byte) (0xff & ((i64 >> 8) ^ 0xff)); + i64out[7] = (byte) (0xff & ((i64) ^ 0xff)); } else { // positive numbers, flip just the first bit - i64out[0] = (byte)(0xff & ((i64 >> 56) ^ 0x80)); - i64out[1] = (byte)(0xff & (i64 >> 48)); - i64out[2] = (byte)(0xff & (i64 >> 40)); - i64out[3] = (byte)(0xff & (i64 >> 32)); - i64out[4] = (byte)(0xff & (i64 >> 24)); - i64out[5] = (byte)(0xff & (i64 >> 16)); - i64out[6] = (byte)(0xff & (i64 >> 8)); - i64out[7] = (byte)(0xff & (i64)); + i64out[0] = (byte) (0xff & ((i64 >> 56) ^ 0x80)); + i64out[1] = (byte) (0xff & (i64 >> 48)); + i64out[2] = (byte) (0xff & (i64 >> 40)); + i64out[3] = (byte) (0xff & (i64 >> 32)); + i64out[4] = (byte) (0xff & (i64 >> 24)); + i64out[5] = (byte) (0xff & (i64 >> 16)); + i64out[6] = (byte) (0xff & (i64 >> 8)); + i64out[7] = (byte) (0xff & (i64)); } writeRawBytes(nonNullByte, 0, 1); writeRawBytes(i64out, 0, 8); } - final protected byte[] nullByte = new byte[] {0}; - final protected byte[] nonNullByte = new byte[] {1}; + final protected byte[] nullByte = new byte[] { 0 }; + final protected byte[] nonNullByte = new byte[] { 1 }; /** - * The escaped byte sequence for the null byte. - * This cannot be changed alone without changing the readString() code. + * The escaped byte sequence for the null byte. This cannot be changed alone + * without changing the readString() code. */ - final protected byte[] escapedNull = new byte[] {1,1}; + final protected byte[] escapedNull = new byte[] { 1, 1 }; /** - * The escaped byte sequence for the "\1" byte. - * This cannot be changed alone without changing the readString() code. + * The escaped byte sequence for the "\1" byte. This cannot be changed alone + * without changing the readString() code. */ - final protected byte[] escapedOne = new byte[] {1,2}; + final protected byte[] escapedOne = new byte[] { 1, 2 }; + + @Override public void writeString(String str) throws TException { byte[] dat; try { @@ -307,6 +344,7 @@ writeTextBytes(dat, 0, dat.length); } + @Override public void writeBinary(byte[] bin) throws TException { if (bin == null) { writeRawBytes(nullByte, 0, 1); @@ -316,52 +354,69 @@ } } + @Override public TMessage readMessageBegin() throws TException { - return new TMessage(); + return new TMessage(); } + @Override public void readMessageEnd() throws TException { } TStruct tstruct = new TStruct(); + + @Override public TStruct readStructBegin() throws TException { stackLevel++; if (stackLevel == 1) { topLevelStructFieldID = 0; - ascending = (topLevelStructFieldID >= sortOrder.length() || sortOrder.charAt(topLevelStructFieldID) != '-'); + ascending = (topLevelStructFieldID >= sortOrder.length() || sortOrder + .charAt(topLevelStructFieldID) != '-'); } else { // is this a null? - // only read the is-null byte for level > 1 because the top-level struct can never be null. - if (readIsNull()) return null; + // only read the is-null byte for level > 1 because the top-level struct + // can never be null. + if (readIsNull()) { + return null; + } } return tstruct; } + @Override public void readStructEnd() throws TException { - stackLevel --; + stackLevel--; } TField f = null; + + @Override public TField readFieldBegin() throws TException { - // slight hack to communicate to DynamicSerDe that the field ids are not being set but things are ordered. - f = new TField("", ORDERED_TYPE, (short)-1); - return f; + // slight hack to communicate to DynamicSerDe that the field ids are not + // being set but things are ordered. + f = new TField("", ORDERED_TYPE, (short) -1); + return f; } + @Override public void readFieldEnd() throws TException { if (stackLevel == 1) { - topLevelStructFieldID ++; - ascending = (topLevelStructFieldID >= sortOrder.length() || sortOrder.charAt(topLevelStructFieldID) != '-'); + topLevelStructFieldID++; + ascending = (topLevelStructFieldID >= sortOrder.length() || sortOrder + .charAt(topLevelStructFieldID) != '-'); } } private TMap tmap = null; + /** - * This method always return the same instance of TMap to avoid creating new instances. - * It is the responsibility of the caller to read the value before calling this method again. + * This method always return the same instance of TMap to avoid creating new + * instances. It is the responsibility of the caller to read the value before + * calling this method again. */ + @Override public TMap readMapBegin() throws TException { - stackLevel ++; + stackLevel++; tmap = new TMap(ORDERED_TYPE, ORDERED_TYPE, readI32()); if (tmap.size == 0 && lastPrimitiveWasNull()) { return null; @@ -369,143 +424,162 @@ return tmap; } + @Override public void readMapEnd() throws TException { - stackLevel --; + stackLevel--; } private TList tlist = null; + /** - * This method always return the same instance of TList to avoid creating new instances. - * It is the responsibility of the caller to read the value before calling this method again. + * This method always return the same instance of TList to avoid creating new + * instances. It is the responsibility of the caller to read the value before + * calling this method again. */ + @Override public TList readListBegin() throws TException { - stackLevel ++; + stackLevel++; tlist = new TList(ORDERED_TYPE, readI32()); if (tlist.size == 0 && lastPrimitiveWasNull()) { return null; - } + } return tlist; } + @Override public void readListEnd() throws TException { - stackLevel --; + stackLevel--; } private TSet set = null; + /** - * This method always return the same instance of TSet to avoid creating new instances. - * It is the responsibility of the caller to read the value before calling this method again. + * This method always return the same instance of TSet to avoid creating new + * instances. It is the responsibility of the caller to read the value before + * calling this method again. */ + @Override public TSet readSetBegin() throws TException { - stackLevel ++; + stackLevel++; set = new TSet(ORDERED_TYPE, readI32()); if (set.size == 0 && lastPrimitiveWasNull()) { return null; - } + } return set; } + @Override public void readSetEnd() throws TException { - stackLevel --; + stackLevel--; } - + // This method takes care of bit-flipping for descending order - // Make this method final to improve performance. + // Make this method final to improve performance. final private int readRawAll(byte[] buf, int off, int len) throws TException { int bytes = trans_.readAll(buf, off, len); if (!ascending) { - for(int i=off; i begin) { - writeRawBytes(bytes, begin, i-begin); + writeRawBytes(bytes, begin, i - begin); } // Write the escaped byte. if (bytes[i] == 0) { @@ -573,19 +650,19 @@ } // Move the pointer to the next byte, since we have written // out the escaped byte in the block above already. - begin = i+1; + begin = i + 1; } } // Write the remaining part of the array if (i > begin) { - writeRawBytes(bytes, begin, i-begin); + writeRawBytes(bytes, begin, i - begin); } // Write the terminating NULL byte writeRawBytes(nullByte, 0, 1); } - + public void writeText(Text text) throws TException { writeTextBytes(text.getBytes(), 0, text.getLength()); } - + } Index: serde/src/java/org/apache/hadoop/hive/serde2/thrift/ThriftByteStreamTypedSerDe.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/thrift/ThriftByteStreamTypedSerDe.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/thrift/ThriftByteStreamTypedSerDe.java (working copy) @@ -26,7 +26,6 @@ import org.apache.hadoop.hive.serde2.SerDeException; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; import org.apache.hadoop.io.Writable; - import org.apache.thrift.TBase; import org.apache.thrift.protocol.TProtocol; import org.apache.thrift.protocol.TProtocolFactory; @@ -37,19 +36,24 @@ protected TIOStreamTransport outTransport, inTransport; protected TProtocol outProtocol, inProtocol; - private void init(TProtocolFactory inFactory, TProtocolFactory outFactory) throws Exception { + private void init(TProtocolFactory inFactory, TProtocolFactory outFactory) + throws Exception { outTransport = new TIOStreamTransport(bos); inTransport = new TIOStreamTransport(bis); outProtocol = outFactory.getProtocol(outTransport); inProtocol = inFactory.getProtocol(inTransport); } - public void initialize(Configuration job, Properties tbl) throws SerDeException { - throw new SerDeException("ThriftByteStreamTypedSerDe is still semi-abstract"); + @Override + public void initialize(Configuration job, Properties tbl) + throws SerDeException { + throw new SerDeException( + "ThriftByteStreamTypedSerDe is still semi-abstract"); } - public ThriftByteStreamTypedSerDe(Type objectType, TProtocolFactory inFactory, - TProtocolFactory outFactory) throws SerDeException { + public ThriftByteStreamTypedSerDe(Type objectType, + TProtocolFactory inFactory, TProtocolFactory outFactory) + throws SerDeException { super(objectType); try { init(inFactory, outFactory); @@ -58,14 +62,16 @@ } } + @Override protected ObjectInspectorFactory.ObjectInspectorOptions getObjectInspectorOptions() { return ObjectInspectorFactory.ObjectInspectorOptions.THRIFT; } - + + @Override public Object deserialize(Writable field) throws SerDeException { Object obj = super.deserialize(field); try { - ((TBase)obj).read(inProtocol); + ((TBase) obj).read(inProtocol); } catch (Exception e) { throw new SerDeException(e); } Index: serde/src/java/org/apache/hadoop/hive/serde2/thrift/WriteNullsProtocol.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/thrift/WriteNullsProtocol.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/thrift/WriteNullsProtocol.java (working copy) @@ -21,23 +21,22 @@ import org.apache.thrift.TException; /** - * An interface for TProtocols that actually write out nulls - - * This should be for all those that don't actually use - * fieldids in the written data like TCTLSeparatedProtocol. + * An interface for TProtocols that actually write out nulls - This should be + * for all those that don't actually use fieldids in the written data like + * TCTLSeparatedProtocol. * */ public interface WriteNullsProtocol { /** - * Was the last primitive read really a NULL. Need - * only be called when the value of the primitive - * was 0. ie the protocol should return 0 on nulls - * and the caller will then check if it was actually null - * For boolean this is false. + * Was the last primitive read really a NULL. Need only be called when the + * value of the primitive was 0. ie the protocol should return 0 on nulls and + * the caller will then check if it was actually null For boolean this is + * false. */ public boolean lastPrimitiveWasNull() throws TException; /** - * Write a null + * Write a null */ public void writeNull() throws TException; Index: serde/src/java/org/apache/hadoop/hive/serde2/SerDeException.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/SerDeException.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/SerDeException.java (working copy) @@ -20,7 +20,7 @@ /** * Generic exception class for SerDes - * + * */ public class SerDeException extends Exception { @@ -42,4 +42,3 @@ super(message, cause); } } - Index: serde/src/java/org/apache/hadoop/hive/serde2/Deserializer.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/Deserializer.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/Deserializer.java (working copy) @@ -18,36 +18,44 @@ package org.apache.hadoop.hive.serde2; +import java.util.Properties; + +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.io.Writable; -import org.apache.hadoop.conf.Configuration; -import java.util.Properties; /** - * HiveDeserializer is used to deserialize the data from hadoop Writable to a + * HiveDeserializer is used to deserialize the data from hadoop Writable to a * custom java object that can be of any type that the developer wants. * - * HiveDeserializer also provides the ObjectInspector which can be used to inspect - * the internal structure of the object (that is returned by deserialize function). - * + * HiveDeserializer also provides the ObjectInspector which can be used to + * inspect the internal structure of the object (that is returned by deserialize + * function). + * */ public interface Deserializer { /** * Initialize the HiveDeserializer. - * @param conf System properties - * @param tbl table properties + * + * @param conf + * System properties + * @param tbl + * table properties * @throws SerDeException */ - public void initialize(Configuration conf, Properties tbl) throws SerDeException; - + public void initialize(Configuration conf, Properties tbl) + throws SerDeException; + /** - * Deserialize an object out of a Writable blob. - * In most cases, the return value of this function will be constant since the function - * will reuse the returned object. - * If the client wants to keep a copy of the object, the client needs to clone the - * returned value by calling ObjectInspectorUtils.getStandardObject(). - * @param blob The Writable object containing a serialized object + * Deserialize an object out of a Writable blob. In most cases, the return + * value of this function will be constant since the function will reuse the + * returned object. If the client wants to keep a copy of the object, the + * client needs to clone the returned value by calling + * ObjectInspectorUtils.getStandardObject(). + * + * @param blob + * The Writable object containing a serialized object * @return A Java object representing the contents in the blob. */ public Object deserialize(Writable blob) throws SerDeException; Index: serde/src/java/org/apache/hadoop/hive/serde2/SerDeUtils.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/SerDeUtils.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/SerDeUtils.java (working copy) @@ -18,14 +18,15 @@ package org.apache.hadoop.hive.serde2; -import java.util.*; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import org.apache.hadoop.hive.common.JavaUtils; -import org.apache.hadoop.hive.serde.Constants; import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.StructField; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; @@ -37,11 +38,9 @@ import org.apache.hadoop.hive.serde2.objectinspector.primitive.LongObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.ShortObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.StringObjectInspector; -import org.apache.hadoop.io.Text; public class SerDeUtils { - public static final char QUOTE = '"'; public static final char COLON = ':'; public static final char COMMA = ','; @@ -50,40 +49,46 @@ public static final String LBRACE = "{"; public static final String RBRACE = "}"; - private static HashMap> serdes = new HashMap> (); + private static HashMap> serdes = new HashMap>(); public static void registerSerDe(String name, Class serde) { - if(serdes.containsKey(name)) { + if (serdes.containsKey(name)) { throw new RuntimeException("double registering serde " + name); } serdes.put(name, serde); } - public static Deserializer lookupDeserializer(String name) throws SerDeException { + public static Deserializer lookupDeserializer(String name) + throws SerDeException { Class c; - if(serdes.containsKey(name)) { - c = serdes.get(name); + if (serdes.containsKey(name)) { + c = serdes.get(name); } else { try { c = Class.forName(name, true, JavaUtils.getClassLoader()); - } catch(ClassNotFoundException e) { + } catch (ClassNotFoundException e) { throw new SerDeException("SerDe " + name + " does not exist"); } } try { - return (Deserializer)c.newInstance(); - } catch(Exception e) { + return (Deserializer) c.newInstance(); + } catch (Exception e) { throw new SerDeException(e); } } - private static List nativeSerDeNames = new ArrayList(); + private static List nativeSerDeNames = new ArrayList(); static { - nativeSerDeNames.add(org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe.class.getName()); - nativeSerDeNames.add(org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe.class.getName()); + nativeSerDeNames + .add(org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe.class + .getName()); + nativeSerDeNames + .add(org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe.class + .getName()); // For backward compatibility nativeSerDeNames.add("org.apache.hadoop.hive.serde.thrift.columnsetSerDe"); - nativeSerDeNames.add(org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe.class.getName()); + nativeSerDeNames + .add(org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe.class.getName()); } public static boolean isNativeSerDe(String serde) { @@ -91,16 +96,23 @@ } private static boolean initCoreSerDes = registerCoreSerDes(); - + protected static boolean registerCoreSerDes() { - // Eagerly load SerDes so they will register their symbolic names even on Lazy Loading JVMs + // Eagerly load SerDes so they will register their symbolic names even on + // Lazy Loading JVMs try { // loading these classes will automatically register the short names - Class.forName(org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe.class.getName()); - Class.forName(org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe.class.getName()); - Class.forName(org.apache.hadoop.hive.serde2.thrift.ThriftDeserializer.class.getName()); + Class + .forName(org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe.class + .getName()); + Class.forName(org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe.class + .getName()); + Class + .forName(org.apache.hadoop.hive.serde2.thrift.ThriftDeserializer.class + .getName()); } catch (ClassNotFoundException e) { - throw new RuntimeException("IMPOSSIBLE Exception: Unable to initialize core serdes", e); + throw new RuntimeException( + "IMPOSSIBLE Exception: Unable to initialize core serdes", e); } return true; } @@ -159,7 +171,6 @@ return (escape.toString()); } - public static String lightEscapeString(String str) { int length = str.length(); StringBuilder escape = new StringBuilder(length + 16); @@ -193,123 +204,129 @@ return sb.toString(); } - static void buildJSONString(StringBuilder sb, Object o, ObjectInspector oi) { - switch(oi.getCategory()) { - case PRIMITIVE: { - PrimitiveObjectInspector poi = (PrimitiveObjectInspector)oi; - if (o == null) { - sb.append("null"); - } else { - switch (poi.getPrimitiveCategory()) { - case BOOLEAN: { - boolean b = ((BooleanObjectInspector)poi).get(o); - sb.append(b ? "true" : "false"); - break; - } - case BYTE: { - sb.append(((ByteObjectInspector)poi).get(o)); - break; - } - case SHORT: { - sb.append(((ShortObjectInspector)poi).get(o)); - break; - } - case INT: { - sb.append(((IntObjectInspector)poi).get(o)); - break; - } - case LONG: { - sb.append(((LongObjectInspector)poi).get(o)); - break; - } - case FLOAT: { - sb.append(((FloatObjectInspector)poi).get(o)); - break; - } - case DOUBLE: { - sb.append(((DoubleObjectInspector)poi).get(o)); - break; - } - case STRING: { - sb.append('"'); - sb.append(escapeString(((StringObjectInspector)poi).getPrimitiveJavaObject(o))); - sb.append('"'); - break; - } - default: - throw new RuntimeException("Unknown primitive type: " + poi.getPrimitiveCategory()); - } + switch (oi.getCategory()) { + case PRIMITIVE: { + PrimitiveObjectInspector poi = (PrimitiveObjectInspector) oi; + if (o == null) { + sb.append("null"); + } else { + switch (poi.getPrimitiveCategory()) { + case BOOLEAN: { + boolean b = ((BooleanObjectInspector) poi).get(o); + sb.append(b ? "true" : "false"); + break; } - break; + case BYTE: { + sb.append(((ByteObjectInspector) poi).get(o)); + break; + } + case SHORT: { + sb.append(((ShortObjectInspector) poi).get(o)); + break; + } + case INT: { + sb.append(((IntObjectInspector) poi).get(o)); + break; + } + case LONG: { + sb.append(((LongObjectInspector) poi).get(o)); + break; + } + case FLOAT: { + sb.append(((FloatObjectInspector) poi).get(o)); + break; + } + case DOUBLE: { + sb.append(((DoubleObjectInspector) poi).get(o)); + break; + } + case STRING: { + sb.append('"'); + sb.append(escapeString(((StringObjectInspector) poi) + .getPrimitiveJavaObject(o))); + sb.append('"'); + break; + } + default: + throw new RuntimeException("Unknown primitive type: " + + poi.getPrimitiveCategory()); + } } - case LIST: { - ListObjectInspector loi = (ListObjectInspector)oi; - ObjectInspector listElementObjectInspector = loi.getListElementObjectInspector(); - List olist = loi.getList(o); - if (olist == null) { - sb.append("null"); - } else { - sb.append(LBRACKET); - for (int i=0; i0) sb.append(COMMA); - buildJSONString(sb, olist.get(i), listElementObjectInspector); + break; + } + case LIST: { + ListObjectInspector loi = (ListObjectInspector) oi; + ObjectInspector listElementObjectInspector = loi + .getListElementObjectInspector(); + List olist = loi.getList(o); + if (olist == null) { + sb.append("null"); + } else { + sb.append(LBRACKET); + for (int i = 0; i < olist.size(); i++) { + if (i > 0) { + sb.append(COMMA); } - sb.append(RBRACKET); + buildJSONString(sb, olist.get(i), listElementObjectInspector); } - break; + sb.append(RBRACKET); } - case MAP: { - MapObjectInspector moi = (MapObjectInspector)oi; - ObjectInspector mapKeyObjectInspector = moi.getMapKeyObjectInspector(); - ObjectInspector mapValueObjectInspector = moi.getMapValueObjectInspector(); - Map omap = moi.getMap(o); - if (omap == null) { - sb.append("null"); - } else { - sb.append(LBRACE); - boolean first = true; - for(Object entry : omap.entrySet()) { - if (first) { - first = false; - } else { - sb.append(COMMA); - } - Map.Entry e = (Map.Entry)entry; - buildJSONString(sb, e.getKey(), mapKeyObjectInspector); - sb.append(COLON); - buildJSONString(sb, e.getValue(), mapValueObjectInspector); + break; + } + case MAP: { + MapObjectInspector moi = (MapObjectInspector) oi; + ObjectInspector mapKeyObjectInspector = moi.getMapKeyObjectInspector(); + ObjectInspector mapValueObjectInspector = moi + .getMapValueObjectInspector(); + Map omap = moi.getMap(o); + if (omap == null) { + sb.append("null"); + } else { + sb.append(LBRACE); + boolean first = true; + for (Object entry : omap.entrySet()) { + if (first) { + first = false; + } else { + sb.append(COMMA); } - sb.append(RBRACE); + Map.Entry e = (Map.Entry) entry; + buildJSONString(sb, e.getKey(), mapKeyObjectInspector); + sb.append(COLON); + buildJSONString(sb, e.getValue(), mapValueObjectInspector); } - break; + sb.append(RBRACE); } - case STRUCT: { - StructObjectInspector soi = (StructObjectInspector)oi; - List structFields = soi.getAllStructFieldRefs(); - if (o == null) { - sb.append("null"); - } else { - sb.append(LBRACE); - for(int i=0; i0) { - sb.append(COMMA); - } - sb.append(QUOTE); - sb.append(structFields.get(i).getFieldName()); - sb.append(QUOTE); - sb.append(COLON); - buildJSONString(sb, soi.getStructFieldData(o, structFields.get(i)), - structFields.get(i).getFieldObjectInspector()); + break; + } + case STRUCT: { + StructObjectInspector soi = (StructObjectInspector) oi; + List structFields = soi.getAllStructFieldRefs(); + if (o == null) { + sb.append("null"); + } else { + sb.append(LBRACE); + for (int i = 0; i < structFields.size(); i++) { + if (i > 0) { + sb.append(COMMA); } - sb.append(RBRACE); + sb.append(QUOTE); + sb.append(structFields.get(i).getFieldName()); + sb.append(QUOTE); + sb.append(COLON); + buildJSONString(sb, soi.getStructFieldData(o, structFields.get(i)), + structFields.get(i).getFieldObjectInspector()); } - break; + sb.append(RBRACE); } - default: - throw new RuntimeException("Unknown type in ObjectInspector!"); - }; - - } + break; + } + default: + throw new RuntimeException("Unknown type in ObjectInspector!"); + } + ; + + } } Index: serde/src/java/org/apache/hadoop/hive/serde2/io/DoubleWritable.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/io/DoubleWritable.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/io/DoubleWritable.java (working copy) @@ -52,30 +52,37 @@ out.writeDouble(value); } - public void set(double value) { this.value = value; } + public void set(double value) { + this.value = value; + } - public double get() { return value; } + public double get() { + return value; + } /** * Returns true iff o is a DoubleWritable with the same value. */ + @Override public boolean equals(Object o) { if (!(o instanceof DoubleWritable)) { return false; } - DoubleWritable other = (DoubleWritable)o; - return this.value == other.value; + DoubleWritable other = (DoubleWritable) o; + return value == other.value; } + @Override public int hashCode() { - return (int)Double.doubleToLongBits(value); + return (int) Double.doubleToLongBits(value); } public int compareTo(Object o) { - DoubleWritable other = (DoubleWritable)o; + DoubleWritable other = (DoubleWritable) o; return (value < other.value ? -1 : (value == other.value ? 0 : 1)); } + @Override public String toString() { return Double.toString(value); } @@ -86,18 +93,16 @@ super(DoubleWritable.class); } - public int compare(byte[] b1, int s1, int l1, - byte[] b2, int s2, int l2) { + @Override + public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) { double thisValue = readDouble(b1, s1); double thatValue = readDouble(b2, s2); return (thisValue < thatValue ? -1 : (thisValue == thatValue ? 0 : 1)); } } - static { // register this comparator + static { // register this comparator WritableComparator.define(DoubleWritable.class, new Comparator()); } } - - Index: serde/src/java/org/apache/hadoop/hive/serde2/io/ShortWritable.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/io/ShortWritable.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/io/ShortWritable.java (working copy) @@ -17,7 +17,6 @@ */ package org.apache.hadoop.hive.serde2.io; - import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; @@ -47,28 +46,31 @@ public void set(short value) { this.value = value; } - + public short get() { return value; } - + + @Override public boolean equals(Object o) { if (o == null || o.getClass() != ShortWritable.class) { return false; } - return get() == ((ShortWritable)o).get(); + return get() == ((ShortWritable) o).get(); } + @Override public int hashCode() { return value; } - + + @Override public String toString() { return String.valueOf(get()); } - + public int compareTo(Object o) { - int thisValue = this.value; + int thisValue = value; int thatValue = ((ShortWritable) o).value; return thisValue - thatValue; } @@ -82,9 +84,10 @@ /** * Compare the buffers in serialized form. */ + @Override public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) { - int a1 = (short)(readUnsignedShort(b1, s1)); - int a2 = (short)(readUnsignedShort(b2, s2)); + int a1 = (short) (readUnsignedShort(b1, s1)); + int a2 = (short) (readUnsignedShort(b2, s2)); return a1 - a2; } } Index: serde/src/java/org/apache/hadoop/hive/serde2/io/ByteWritable.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/io/ByteWritable.java (revision 901519) +++ serde/src/java/org/apache/hadoop/hive/serde2/io/ByteWritable.java (working copy) @@ -17,7 +17,6 @@ */ package org.apache.hadoop.hive.serde2.io; - import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; @@ -47,33 +46,36 @@ public void set(byte value) { this.value = value; } - + public byte get() { return value; } - + /** Compares two ByteWritables. */ public int compareTo(Object o) { - int thisValue = this.value; + int thisValue = value; int thatValue = ((ByteWritable) o).value; return thisValue - thatValue; } + @Override public boolean equals(Object o) { if (o == null || o.getClass() != ByteWritable.class) { return false; } - return get() == ((ByteWritable)o).get(); + return get() == ((ByteWritable) o).get(); } - + + @Override public int hashCode() { return value; } - + + @Override public String toString() { return String.valueOf(get()); } - + /** A Comparator optimized for BytesWritable. */ public static class Comparator extends WritableComparator { public Comparator() { @@ -83,6 +85,7 @@ /** * Compare the buffers in serialized form. */ + @Override public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) { int a1 = b1[s1]; int a2 = b2[s2];