Index: src/contrib/hive/serde/src/gen-py/org_apache_hadoop_hive_serde/constants.py =================================================================== --- src/contrib/hive/serde/src/gen-py/org_apache_hadoop_hive_serde/constants.py (revision 711472) +++ src/contrib/hive/serde/src/gen-py/org_apache_hadoop_hive_serde/constants.py (working copy) @@ -15,6 +15,10 @@ SERIALIZATION_DDL = 'serialization.ddl' +SERIALIZATION_NULL_FORMAT = 'serialization.null.format' + +SERIALIZATION_SORT_ORDER = 'serialization.sort.order' + FIELD_DELIM = 'field.delim' COLLECTION_DELIM = 'colelction.delim' @@ -23,8 +27,12 @@ MAPKEY_DELIM = 'mapkey.delim' +BOOLEAN_TYPE_NAME = 'boolean' + TINYINT_TYPE_NAME = 'tinyint' +SMALLINT_TYPE_NAME = 'smallint' + INT_TYPE_NAME = 'int' BIGINT_TYPE_NAME = 'bigint' @@ -46,7 +54,9 @@ MAP_TYPE_NAME = 'map' PrimitiveTypes = set([ + 'boolean', 'tinyint', + 'smallint', 'int', 'bigint', 'float', Index: src/contrib/hive/serde/src/test/org/apache/hadoop/hive/serde2/TestTCTLSeparatedProtocol.java =================================================================== --- src/contrib/hive/serde/src/test/org/apache/hadoop/hive/serde2/TestTCTLSeparatedProtocol.java (revision 711472) +++ src/contrib/hive/serde/src/test/org/apache/hadoop/hive/serde2/TestTCTLSeparatedProtocol.java (working copy) @@ -242,4 +242,106 @@ } } + + + public void testNulls() throws Exception { + try { + TMemoryBuffer trans = new TMemoryBuffer(1024); + TCTLSeparatedProtocol prot = new TCTLSeparatedProtocol(trans, 3); + + prot.writeStructBegin(new TStruct()); + + prot.writeFieldBegin(new TField()); + prot.writeString(null); + prot.writeFieldEnd(); + + prot.writeFieldBegin(new TField()); + prot.writeString(null); + prot.writeFieldEnd(); + + prot.writeFieldBegin(new TField()); + prot.writeI32(100); + prot.writeFieldEnd(); + + prot.writeFieldBegin(new TField()); + prot.writeString(null); + prot.writeFieldEnd(); + + prot.writeFieldBegin(new TField()); + prot.writeMapBegin(new TMap()); + prot.writeString(null); + prot.writeString(null); + prot.writeString("key2"); + prot.writeString(null); + prot.writeString(null); + prot.writeString("val3"); + prot.writeMapEnd(); + prot.writeFieldEnd(); + + prot.writeStructEnd(); + + byte b[] = new byte[3*1024]; + int len = trans.read(b,0,b.length); + String written = new String(b,0,len); + + String testRef = "\\N\\N100\\N\\N\\Nkey2\\N\\Nval3"; + + assertTrue(testRef.equals(written)); + + trans = new TMemoryBuffer(1023); + trans.write(b, 0, len); + + prot = new TCTLSeparatedProtocol(trans, 3); + + prot.readStructBegin(); + + prot.readFieldBegin(); + String ret = prot.readString(); + prot.readFieldEnd(); + + assertTrue(ret.equals("")); + + prot.readFieldBegin(); + ret = prot.readString(); + prot.readFieldEnd(); + + assertTrue(ret.equals("")); + + prot.readFieldBegin(); + int ret1 = prot.readI32(); + prot.readFieldEnd(); + + assertTrue(ret1 == 100); + + + prot.readFieldBegin(); + ret1 = prot.readI32(); + prot.readFieldEnd(); + + prot.readFieldBegin(); + TMap map = prot.readMapBegin(); + + assertTrue(map.size == 3); + + assertTrue(prot.readString().isEmpty()); + assertTrue(prot.readString().isEmpty()); + + assertTrue(prot.readString().equals("key2")); + assertTrue(prot.readString().isEmpty()); + + assertTrue(prot.readString().isEmpty()); + assertTrue(prot.readString().equals("val3")); + + prot.readMapEnd(); + prot.readFieldEnd(); + + assertTrue(ret1 == 0); + + } catch(Exception e) { + e.printStackTrace(); + } + } + + + } Index: src/contrib/hive/serde/src/test/org/apache/hadoop/hive/serde2/dynamic_type/TestDynamicSerDe.java =================================================================== --- src/contrib/hive/serde/src/test/org/apache/hadoop/hive/serde2/dynamic_type/TestDynamicSerDe.java (revision 711472) +++ src/contrib/hive/serde/src/test/org/apache/hadoop/hive/serde2/dynamic_type/TestDynamicSerDe.java (working copy) @@ -20,8 +20,11 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; +import java.util.Map; import java.util.List; import java.util.Properties; +import java.util.Random; +import java.util.Map.Entry; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.serde.Constants; @@ -35,6 +38,13 @@ public class TestDynamicSerDe extends TestCase { + public static HashMap makeHashMap(String... params) { + HashMap r = new HashMap(); + for(int i=0; i protocols = new ArrayList(); ArrayList isBinaries = new ArrayList(); - + ArrayList> additionalParams = new ArrayList>(); + + protocols.add(org.apache.hadoop.hive.serde2.thrift.TBinarySortableProtocol.class.getName()); + isBinaries.add(true); + additionalParams.add(makeHashMap("serialization.sort.order", "++++++")); + protocols.add(org.apache.hadoop.hive.serde2.thrift.TBinarySortableProtocol.class.getName()); + isBinaries.add(true); + additionalParams.add(makeHashMap("serialization.sort.order", "------")); + + protocols.add(com.facebook.thrift.protocol.TBinaryProtocol.class.getName()); isBinaries.add(true); + additionalParams.add(null); protocols.add(com.facebook.thrift.protocol.TJSONProtocol.class.getName()); isBinaries.add(false); + additionalParams.add(null); // TSimpleJSONProtocol does not support deserialization. // protocols.add(com.facebook.thrift.protocol.TSimpleJSONProtocol.class.getName()); // isBinaries.add(false); + // additionalParams.add(null); // TCTLSeparatedProtocol is not done yet. protocols.add(org.apache.hadoop.hive.serde2.thrift.TCTLSeparatedProtocol.class.getName()); isBinaries.add(false); + additionalParams.add(null); System.out.println("input struct = " + struct); @@ -80,8 +107,14 @@ schema.setProperty(Constants.SERIALIZATION_FORMAT, protocol); schema.setProperty(org.apache.hadoop.hive.metastore.api.Constants.META_TABLE_NAME, "test"); schema.setProperty(Constants.SERIALIZATION_DDL, - "struct test { i32 hello, list bye, map another}"); + "struct test { i32 hello, list bye, map another, i32 nhello, double d, double nd}"); schema.setProperty(Constants.SERIALIZATION_LIB, new DynamicSerDe().getClass().toString()); + HashMap p = additionalParams.get(pp); + if (p != null) { + for(Entry e: p.entrySet()) { + schema.setProperty(e.getKey(), e.getValue()); + } + } DynamicSerDe serde = new DynamicSerDe(); serde.initialize(new Configuration(), schema); @@ -93,15 +126,8 @@ // Try to serialize BytesWritable bytes = (BytesWritable) serde.serialize(struct, oi); + System.out.println("bytes =" + hexString(bytes)); - StringBuilder sb = new StringBuilder(); - for (int i=0; i0) { + int compareResult = bytes[i-1].compareTo(bytes[i]); + if ( (compareResult<0 && !ascending) || (compareResult>0 && ascending) ) { + System.out.println("Test failed in " + (ascending ? "ascending" : "descending") + " order."); + System.out.println("serialized data of " + structs[i-1] + " = " + hexString(bytes[i-1])); + System.out.println("serialized data of " + structs[i] + " = " + hexString(bytes[i])); + fail("Sort order of serialized " + structs[i-1] + " and " + structs[i] + " are reversed!"); + } + } + } + + // Try to deserialize + Object[] deserialized = new Object[structs.length]; + for (int i=0; i struct = new ArrayList(); + if (i==0) { + struct.add(null); + } else { + struct.add(Double.valueOf((r.nextDouble()-0.5)*10)); + } + structs[i] = struct; + } + sort(structs); + ddl = "struct test { double hello}"; + System.out.println("Testing " + ddl); + testTBinarySortableProtocol(structs, ddl, true); + testTBinarySortableProtocol(structs, ddl, false); + + // Test integer + for (int i=0; i struct = new ArrayList(); + if (i==0) { + struct.add(null); + } else { + struct.add((int)((r.nextDouble()-0.5)*1.5*Integer.MAX_VALUE)); + } + structs[i] = struct; + } + sort(structs); + // Null should be smaller than any other value, so put a null at the front end + // to test whether that is held. + ((List)structs[0]).set(0, null); + ddl = "struct test { i32 hello}"; + System.out.println("Testing " + ddl); + testTBinarySortableProtocol(structs, ddl, true); + testTBinarySortableProtocol(structs, ddl, false); + + // Test long + for (int i=0; i struct = new ArrayList(); + if (i==0) { + struct.add(null); + } else { + struct.add((long)((r.nextDouble()-0.5)*1.5*Long.MAX_VALUE)); + } + structs[i] = struct; + } + sort(structs); + // Null should be smaller than any other value, so put a null at the front end + // to test whether that is held. + ((List)structs[0]).set(0, null); + ddl = "struct test { i64 hello}"; + System.out.println("Testing " + ddl); + testTBinarySortableProtocol(structs, ddl, true); + testTBinarySortableProtocol(structs, ddl, false); + + // Test string + for (int i=0; i struct = new ArrayList(); + if (i==0) { + struct.add(null); + } else { + struct.add(String.valueOf((r.nextDouble()-0.5)*1000)); + } + structs[i] = struct; + } + sort(structs); + // Null should be smaller than any other value, so put a null at the front end + // to test whether that is held. + ((List)structs[0]).set(0, null); + ddl = "struct test { string hello}"; + System.out.println("Testing " + ddl); + testTBinarySortableProtocol(structs, ddl, true); + testTBinarySortableProtocol(structs, ddl, false); + + // Test string + double + for (int i=0; i struct = new ArrayList(); + if (i%9==0) { + struct.add(null); + } else { + struct.add("str" + (i/5)); + } + if (i%7==0) { + struct.add(null); + } else { + struct.add(Double.valueOf((r.nextDouble()-0.5)*10)); + } + structs[i] = struct; + } + sort(structs); + // Null should be smaller than any other value, so put a null at the front end + // to test whether that is held. + ((List)structs[0]).set(0, null); + ddl = "struct test { string hello, double another}"; + System.out.println("Testing " + ddl); + testTBinarySortableProtocol(structs, ddl, true); + testTBinarySortableProtocol(structs, ddl, false); + + System.out.println("Test testTBinarySortableProtocol passed!"); + } catch (Throwable e) { + e.printStackTrace(); + throw e; + } + } + + public void testConfigurableTCTLSeparated() throws Throwable { try { @@ -168,13 +401,7 @@ // Try to serialize BytesWritable bytes = (BytesWritable) serde.serialize(struct, oi); - StringBuilder sb = new StringBuilder(); - for (int i=0; i bye = null; + HashMap another = new HashMap(); + another.put("firstKey", 1); + another.put("secondKey", 2); + ArrayList struct = new ArrayList(); + struct.add(Integer.valueOf(234)); + struct.add(bye); + struct.add(another); + + Properties schema = new Properties(); + schema.setProperty(Constants.SERIALIZATION_FORMAT, org.apache.hadoop.hive.serde2.thrift.TCTLSeparatedProtocol.class.getName()); + schema.setProperty(org.apache.hadoop.hive.metastore.api.Constants.META_TABLE_NAME, "test"); + schema.setProperty(Constants.SERIALIZATION_DDL, + "struct test { i32 hello, list bye, map another}"); + schema.setProperty(Constants.SERIALIZATION_LIB, new DynamicSerDe().getClass().toString()); + schema.setProperty(TCTLSeparatedProtocol.ReturnNullsKey, "true"); + + DynamicSerDe serde = new DynamicSerDe(); + serde.initialize(new Configuration(), schema); + + ObjectInspector oi = serde.getObjectInspector(); + + // Try to serialize + BytesWritable bytes = (BytesWritable) serde.serialize(struct, oi); + + hexString(bytes); + + // Try to deserialize + Object o = serde.deserialize(bytes); + assertEquals(struct, o); + + } catch (Throwable e) { + e.printStackTrace(); + throw e; + } + + } + + /** + * Tests all elements of a struct being null with return nulls on + */ + + public void testNulls2() throws Throwable { + try { + + + // Try to construct an object + ArrayList bye = null; + HashMap another = null; + ArrayList struct = new ArrayList(); + struct.add(null); + struct.add(bye); + struct.add(another); + + Properties schema = new Properties(); + schema.setProperty(Constants.SERIALIZATION_FORMAT, org.apache.hadoop.hive.serde2.thrift.TCTLSeparatedProtocol.class.getName()); + schema.setProperty(org.apache.hadoop.hive.metastore.api.Constants.META_TABLE_NAME, "test"); + schema.setProperty(Constants.SERIALIZATION_DDL, + "struct test { i32 hello, list bye, map another}"); + schema.setProperty(Constants.SERIALIZATION_LIB, new DynamicSerDe().getClass().toString()); + schema.setProperty(TCTLSeparatedProtocol.ReturnNullsKey, "true"); + + DynamicSerDe serde = new DynamicSerDe(); + serde.initialize(new Configuration(), schema); + + ObjectInspector oi = serde.getObjectInspector(); + + // Try to serialize + BytesWritable bytes = (BytesWritable) serde.serialize(struct, oi); + + hexString(bytes); + + // Try to deserialize + Object o = serde.deserialize(bytes); + List olist = (List)o; + + assertTrue(olist.size() == 3); + assertEquals(null, olist.get(0)); + assertEquals(null, olist.get(1)); + assertEquals(null, olist.get(2)); + + // assertEquals(o, struct); Cannot do this because types of null lists are wrong. + + } catch (Throwable e) { + e.printStackTrace(); + throw e; + } + + } + + /** + * Tests map and list being empty with return nulls on + */ + + public void testNulls3() throws Throwable { + try { + + + // Try to construct an object + ArrayList bye = new ArrayList (); + HashMap another = null; + ArrayList struct = new ArrayList(); + struct.add(null); + struct.add(bye); + struct.add(another); + + Properties schema = new Properties(); + schema.setProperty(Constants.SERIALIZATION_FORMAT, org.apache.hadoop.hive.serde2.thrift.TCTLSeparatedProtocol.class.getName()); + schema.setProperty(org.apache.hadoop.hive.metastore.api.Constants.META_TABLE_NAME, "test"); + schema.setProperty(Constants.SERIALIZATION_DDL, + "struct test { i32 hello, list bye, map another}"); + schema.setProperty(Constants.SERIALIZATION_LIB, new DynamicSerDe().getClass().toString()); + + schema.setProperty(TCTLSeparatedProtocol.ReturnNullsKey, "true"); + DynamicSerDe serde = new DynamicSerDe(); + serde.initialize(new Configuration(), schema); + + ObjectInspector oi = serde.getObjectInspector(); + + // Try to serialize + BytesWritable bytes = (BytesWritable) serde.serialize(struct, oi); + + hexString(bytes); + + // Try to deserialize + Object o = serde.deserialize(bytes); + List olist = (List)o; + + assertTrue(olist.size() == 3); + assertEquals(null, olist.get(0)); + assertEquals(0, ((List)olist.get(1)).size()); + assertEquals(null, olist.get(2)); + + // assertEquals(o, struct); Cannot do this because types of null lists are wrong. + + } catch (Throwable e) { + e.printStackTrace(); + throw e; + } + + } + + + /** + * Tests map and list null/empty with return nulls *off* + */ + + public void testNulls4() throws Throwable { + try { + + + // Try to construct an object + ArrayList bye = new ArrayList (); + HashMap another = null; + ArrayList struct = new ArrayList(); + struct.add(null); + struct.add(bye); + struct.add(another); + + Properties schema = new Properties(); + schema.setProperty(Constants.SERIALIZATION_FORMAT, org.apache.hadoop.hive.serde2.thrift.TCTLSeparatedProtocol.class.getName()); + schema.setProperty(org.apache.hadoop.hive.metastore.api.Constants.META_TABLE_NAME, "test"); + schema.setProperty(Constants.SERIALIZATION_DDL, + "struct test { i32 hello, list bye, map another}"); + schema.setProperty(Constants.SERIALIZATION_LIB, new DynamicSerDe().getClass().toString()); + + schema.setProperty(TCTLSeparatedProtocol.ReturnNullsKey, "false"); + DynamicSerDe serde = new DynamicSerDe(); + serde.initialize(new Configuration(), schema); + + ObjectInspector oi = serde.getObjectInspector(); + + // Try to serialize + BytesWritable bytes = (BytesWritable) serde.serialize(struct, oi); + + hexString(bytes); + + // Try to deserialize + Object o = serde.deserialize(bytes); + List olist = (List)o; + + assertTrue(olist.size() == 3); + assertEquals(new Integer(0), (Integer)olist.get(0)); + List num1 = (List)olist.get(1); + assertTrue(num1.size() == 0); + Map num2 = (Map)olist.get(2); + assertTrue(num2.size() == 0); + + // assertEquals(o, struct); Cannot do this because types of null lists are wrong. + + } catch (Throwable e) { + e.printStackTrace(); + throw e; + } + + } + + + + } Index: src/contrib/hive/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestStandardObjectInspectors.java =================================================================== --- src/contrib/hive/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestStandardObjectInspectors.java (revision 711472) +++ src/contrib/hive/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestStandardObjectInspectors.java (working copy) @@ -34,7 +34,7 @@ assertEquals(oi1, oi2); assertEquals(Category.PRIMITIVE, oi1.getCategory()); assertEquals(c, oi1.getPrimitiveClass()); - assertEquals(ObjectInspectorUtils.getClassShortName(c.getName()), + assertEquals(ObjectInspectorUtils.getClassShortName(c), oi1.getTypeName()); } catch (Throwable e) { e.printStackTrace(); Index: src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde/dynamic_type/DynamicSerDeTypeList.java =================================================================== --- src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde/dynamic_type/DynamicSerDeTypeList.java (revision 711472) +++ src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde/dynamic_type/DynamicSerDeTypeList.java (working copy) @@ -37,57 +37,63 @@ public class DynamicSerDeTypeList extends DynamicSerDeTypeBase { - public boolean isPrimitive() { return false; } - public boolean isList() { return true; } + public boolean isPrimitive() { return false; } + public boolean isList() { return true; } - // production is: list + // production is: list - static final private int FD_TYPE = 0; + static final private int FD_TYPE = 0; - public Class getRealType() { - return java.util.ArrayList.class; - } + public Class getRealType() { + return java.util.ArrayList.class; + } - public DynamicSerDeTypeList(int i) { - super(i); - } - public DynamicSerDeTypeList(thrift_grammar p, int i) { - super(p,i); - } + public DynamicSerDeTypeList(int i) { + super(i); + } + public DynamicSerDeTypeList(thrift_grammar p, int i) { + super(p,i); + } - public DynamicSerDeTypeBase getElementType() { - return (DynamicSerDeTypeBase)((DynamicSerDeFieldType)this.jjtGetChild(FD_TYPE)).getMyType(); - } + public DynamicSerDeTypeBase getElementType() { + return (DynamicSerDeTypeBase)((DynamicSerDeFieldType)this.jjtGetChild(FD_TYPE)).getMyType(); + } - public String toString() { - return "list<" + this.getElementType().toString() + ">"; - } + public String toString() { + return "list<" + this.getElementType().toString() + ">"; + } - public ArrayList deserialize(TProtocol iprot) throws SerDeException, TException, IllegalAccessException { - TList thelist = iprot.readListBegin(); - ArrayList result = new ArrayList () ; - for(int i = 0; i < thelist.size; i++) { - Object elem = this.getElementType().deserialize(iprot); - result.add(elem); - } - // in theory, the below call isn't needed in non thrift_mode, but let's not get too crazy - iprot.readListEnd(); - return result; + public ArrayList deserialize(TProtocol iprot) throws SerDeException, TException, IllegalAccessException { + TList thelist = iprot.readListBegin(); + // thelist might be reused by the TProtocol + int listSize = thelist.size; + ArrayList result = new ArrayList () ; + for(int i = 0; i < listSize; i++) { + Object elem = this.getElementType().deserialize(iprot); + result.add(elem); } + // in theory, the below call isn't needed in non thrift_mode, but let's not get too crazy + iprot.readListEnd(); + return result; + } - public void serialize(Object obj, TProtocol oprot) throws TException, SerDeException, NoSuchFieldException,IllegalAccessException { - List list = (List)obj; - DynamicSerDeTypeBase mt = this.getElementType(); - oprot.writeListBegin(new TList(mt.getType(),list.size())); - for(Object o: list) { - mt.serialize(o,oprot); - } - // in theory, the below call isn't needed in non thrift_mode, but let's not get too crazy - oprot.writeListEnd(); - } + TList serializeList = new TList(); + public void serialize(Object obj, TProtocol oprot) throws TException, SerDeException, NoSuchFieldException,IllegalAccessException { + List list = (List)obj; + DynamicSerDeTypeBase mt = this.getElementType(); - public byte getType() { - return TType.LIST; + serializeList.elemType = mt.getType(); + serializeList.size = list.size(); + oprot.writeListBegin(serializeList); + for (Object o: list) { + mt.serialize(o,oprot); } + // in theory, the below call isn't needed in non thrift_mode, but let's not get too crazy + oprot.writeListEnd(); + } + public byte getType() { + return TType.LIST; + } + } Index: src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/MetadataTypedColumnsetSerDe.java =================================================================== --- src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/MetadataTypedColumnsetSerDe.java (revision 711472) +++ src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/MetadataTypedColumnsetSerDe.java (working copy) @@ -103,7 +103,7 @@ if (columnProperty == null || columnProperty.length() == 0 || columnsetSerDe) { // Hack for tables with no columns - // Treat it as a table with a single column called "col" + // Treat it as a table with a single column called "col" cachedObjectInspector = ObjectInspectorFactory.getReflectionObjectInspector( ColumnSet.class, ObjectInspectorFactory.ObjectInspectorOptions.JAVA); } else { Index: src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypeString.java =================================================================== --- src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypeString.java (revision 711472) +++ src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypeString.java (working copy) @@ -47,14 +47,14 @@ public String toString() { return "string"; } public String deserialize(TProtocol iprot) throws SerDeException, TException, IllegalAccessException { - return String.valueOf(iprot.readString()); + return iprot.readString(); } @Override public Object deserialize(Object reuse, TProtocol iprot) throws SerDeException, TException, IllegalAccessException { - return String.valueOf(iprot.readString()); + return iprot.readString(); } - + public void serialize(Object s, TProtocol oprot) throws TException, SerDeException, NoSuchFieldException,IllegalAccessException { oprot.writeString((String)s); } Index: src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypeDouble.java =================================================================== --- src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypeDouble.java (revision 711472) +++ src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypeDouble.java (working copy) @@ -46,7 +46,12 @@ public String toString() { return "double"; } public Object deserialize(Object reuse, TProtocol iprot) throws SerDeException, TException, IllegalAccessException { - return Double.valueOf(iprot.readDouble()); + double val = iprot.readDouble(); + if (val == 0 && iprot instanceof org.apache.hadoop.hive.serde2.thrift.WriteNullsProtocol && + ((org.apache.hadoop.hive.serde2.thrift.WriteNullsProtocol)iprot).lastPrimitiveWasNull()) { + return null; + } + return Double.valueOf(val); } public void serialize(Object o, ObjectInspector oi, TProtocol oprot) throws TException, SerDeException, NoSuchFieldException,IllegalAccessException { @@ -58,4 +63,7 @@ public byte getType() { return TType.DOUBLE; } + + public Class getRealType() { return java.lang.Double.class; } + public Double getRealTypeInstance() { return Double.valueOf(0); } } Index: src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypeList.java =================================================================== --- src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypeList.java (revision 711472) +++ src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypeList.java (working copy) @@ -59,6 +59,10 @@ @Override public ArrayList deserialize(Object reuse, TProtocol iprot) throws SerDeException, TException, IllegalAccessException { TList thelist = iprot.readListBegin(); + if (thelist == null) { + return null; + } + ArrayList deserializeReuse; if (reuse != null) { deserializeReuse = (ArrayList)reuse; @@ -89,17 +93,32 @@ ObjectInspector elementObjectInspector = loi.getListElementObjectInspector(); DynamicSerDeTypeBase mt = this.getElementType(); + org.apache.hadoop.hive.serde2.thrift.WriteNullsProtocol nullProtocol = + (oprot instanceof org.apache.hadoop.hive.serde2.thrift.WriteNullsProtocol) + ? (org.apache.hadoop.hive.serde2.thrift.WriteNullsProtocol)oprot + : null; + if (o instanceof List) { List list = (List)o; oprot.writeListBegin(new TList(mt.getType(),list.size())); for (Object element: list) { - mt.serialize(element, elementObjectInspector, oprot); + if (element == null) { + assert(nullProtocol != null); + nullProtocol.writeNull(); + } else { + mt.serialize(element, elementObjectInspector, oprot); + } } } else { Object[] list = (Object[])o; oprot.writeListBegin(new TList(mt.getType(),list.length)); for (Object element: list) { - mt.serialize(element, elementObjectInspector, oprot); + if (element == null && nullProtocol != null) { + assert(nullProtocol != null); + nullProtocol.writeNull(); + } else { + mt.serialize(element, elementObjectInspector, oprot); + } } } // in theory, the below call isn't needed in non thrift_mode, but let's not get too crazy Index: src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeFieldList.java =================================================================== --- src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeFieldList.java (revision 711472) +++ src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeFieldList.java (working copy) @@ -174,6 +174,8 @@ assert(oi instanceof StructObjectInspector); StructObjectInspector soi = (StructObjectInspector) oi; + boolean writeNulls = oprot instanceof org.apache.hadoop.hive.serde2.thrift.WriteNullsProtocol; + // For every field List fields = soi.getAllStructFieldRefs(); if (fields.size() != ordered_types.length) { @@ -184,6 +186,10 @@ Object f = soi.getStructFieldData(o, fields.get(i)); DynamicSerDeTypeBase mt = ordered_types[i]; + if (f == null && !writeNulls) { + continue; + } + if(thrift_mode) { field.name = mt.name; field.type = mt.getType(); @@ -191,8 +197,11 @@ oprot.writeFieldBegin(field); } - mt.serialize(f, fields.get(i).getFieldObjectInspector(), oprot); - + if(f == null) { + ((org.apache.hadoop.hive.serde2.thrift.WriteNullsProtocol)oprot).writeNull(); + } else { + mt.serialize(f, fields.get(i).getFieldObjectInspector(), oprot); + } if(thrift_mode) { oprot.writeFieldEnd(); } Index: src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypeMap.java =================================================================== --- src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypeMap.java (revision 711472) +++ src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypeMap.java (working copy) @@ -86,7 +86,12 @@ deserializeReuse = new HashMap(); } TMap themap = iprot.readMapBegin(); - for(int i = 0; i < themap.size; i++) { + if (themap == null) { + return null; + } + // themap might be reused by the Protocol. + int mapSize = themap.size; + for(int i = 0; i < mapSize; i++) { Object key = this.getKeyType().deserialize(null, iprot); Object value = this.getValueType().deserialize(null, iprot); deserializeReuse.put(key,value); @@ -97,6 +102,7 @@ return deserializeReuse; } + TMap serializeMap = new TMap(); @Override public void serialize(Object o, ObjectInspector oi, TProtocol oprot) throws TException, SerDeException, NoSuchFieldException, @@ -104,19 +110,33 @@ DynamicSerDeTypeBase keyType = this.getKeyType(); DynamicSerDeTypeBase valueType = this.getValueType(); + org.apache.hadoop.hive.serde2.thrift.WriteNullsProtocol nullProtocol = + (oprot instanceof org.apache.hadoop.hive.serde2.thrift.WriteNullsProtocol) + ? (org.apache.hadoop.hive.serde2.thrift.WriteNullsProtocol)oprot + : null; + assert(oi.getCategory() == ObjectInspector.Category.MAP); MapObjectInspector moi = (MapObjectInspector)oi; ObjectInspector koi = moi.getMapKeyObjectInspector(); ObjectInspector voi = moi.getMapValueObjectInspector(); Map map = moi.getMap(o); - oprot.writeMapBegin(new TMap(keyType.getType(),valueType.getType(),map.size())); + serializeMap.size = map.size(); + serializeMap.keyType = keyType.getType(); + serializeMap.valueType = valueType.getType(); + oprot.writeMapBegin(serializeMap); + for(Iterator i = map.entrySet().iterator(); i.hasNext(); ) { Map.Entry it = (Map.Entry)i.next(); Object key = it.getKey(); Object value = it.getValue(); keyType.serialize(key, koi, oprot); - valueType.serialize(value, voi, oprot); + if (value == null) { + assert(nullProtocol != null); + nullProtocol.writeNull(); + } else { + valueType.serialize(value, voi, oprot); + } } // in theory, the below call isn't needed in non thrift_mode, but let's not get too crazy oprot.writeMapEnd(); Index: src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypeBool.java =================================================================== --- src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypeBool.java (revision 711472) +++ src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypeBool.java (working copy) @@ -45,7 +45,12 @@ @Override public Object deserialize(Object reuse, TProtocol iprot) throws SerDeException, TException, IllegalAccessException { - return Boolean.valueOf(iprot.readBool()); + boolean val = iprot.readBool(); + if (val == false && iprot instanceof org.apache.hadoop.hive.serde2.thrift.WriteNullsProtocol && + ((org.apache.hadoop.hive.serde2.thrift.WriteNullsProtocol)iprot).lastPrimitiveWasNull()) { + return null; + } + return Boolean.valueOf(val); } @Override @@ -61,4 +66,8 @@ public byte getType() { return TType.BOOL; } + + public Class getRealType() { return java.lang.Boolean.class; } + public Boolean getRealTypeInstance() { return Boolean.FALSE; } + } Index: src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypeSet.java =================================================================== --- src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypeSet.java (revision 711472) +++ src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypeSet.java (working copy) @@ -76,6 +76,9 @@ public Object deserialize(Object reuse, TProtocol iprot) throws SerDeException, TException, IllegalAccessException { TSet theset = iprot.readSetBegin(); + if (theset == null) { + return null; + } Set result; if (reuse != null) { result = (Set)reuse; @@ -96,6 +99,7 @@ * The code uses ListObjectInspector right now. We need to change it to * SetObjectInspector when that is done. */ + TSet tset = new TSet(); @Override public void serialize(Object o, ObjectInspector oi, TProtocol oprot) throws TException, SerDeException, NoSuchFieldException, @@ -105,7 +109,9 @@ Set set = (Set)o; DynamicSerDeTypeBase mt = this.getElementType(); - oprot.writeSetBegin(new TSet(mt.getType(),set.size())); + tset.elemType = mt.getType(); + tset.size = set.size(); + oprot.writeSetBegin(tset); for(Object element: set) { mt.serialize(element, loi.getListElementObjectInspector(), oprot); } Index: src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypei32.java =================================================================== --- src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypei32.java (revision 711472) +++ src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypei32.java (working copy) @@ -47,7 +47,12 @@ @Override public Object deserialize(Object reuse, TProtocol iprot) throws SerDeException, TException, IllegalAccessException { - return Integer.valueOf(iprot.readI32()); + int val = iprot.readI32(); + if (val == 0 && iprot instanceof org.apache.hadoop.hive.serde2.thrift.WriteNullsProtocol && + ((org.apache.hadoop.hive.serde2.thrift.WriteNullsProtocol)iprot).lastPrimitiveWasNull()) { + return null; + } + return Integer.valueOf(val); } @Override Index: src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypei16.java =================================================================== --- src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypei16.java (revision 711472) +++ src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypei16.java (working copy) @@ -49,7 +49,12 @@ @Override public Object deserialize(Object reuse, TProtocol iprot) throws SerDeException, TException, IllegalAccessException { - return Integer.valueOf(iprot.readI16()); + int val = iprot.readI16(); + if (val == 0 && iprot instanceof org.apache.hadoop.hive.serde2.thrift.WriteNullsProtocol && + ((org.apache.hadoop.hive.serde2.thrift.WriteNullsProtocol)iprot).lastPrimitiveWasNull()) { + return null; + } + return Integer.valueOf(val); } @Override Index: src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypei64.java =================================================================== --- src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypei64.java (revision 711472) +++ src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypei64.java (working copy) @@ -49,7 +49,12 @@ @Override public Object deserialize(Object reuse, TProtocol iprot) throws SerDeException, TException, IllegalAccessException { - return Long.valueOf(iprot.readI64()); + long val = iprot.readI64(); + if (val == 0 && iprot instanceof org.apache.hadoop.hive.serde2.thrift.WriteNullsProtocol && + ((org.apache.hadoop.hive.serde2.thrift.WriteNullsProtocol)iprot).lastPrimitiveWasNull()) { + return null; + } + return Long.valueOf(val); } @Override Index: src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypeByte.java =================================================================== --- src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypeByte.java (revision 711472) +++ src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypeByte.java (working copy) @@ -41,10 +41,15 @@ public String toString() { return "byte"; } public Byte deserialize(TProtocol iprot) throws SerDeException, TException, IllegalAccessException { - return Byte.valueOf(iprot.readByte()); + byte val = iprot.readByte(); + if (val == 0 && iprot instanceof org.apache.hadoop.hive.serde2.thrift.WriteNullsProtocol && + ((org.apache.hadoop.hive.serde2.thrift.WriteNullsProtocol)iprot).lastPrimitiveWasNull()) { + return null; + } + return Byte.valueOf(val); } public Object deserialize(Object reuse, TProtocol iprot) throws SerDeException, TException, IllegalAccessException { - return Byte.valueOf(iprot.readByte()); + return deserialize(iprot); } public void serialize(Object s, TProtocol oprot) throws TException, SerDeException, NoSuchFieldException,IllegalAccessException { Index: src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StandardPrimitiveObjectInspector.java =================================================================== --- src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StandardPrimitiveObjectInspector.java (revision 711472) +++ src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StandardPrimitiveObjectInspector.java (working copy) @@ -43,7 +43,7 @@ } public String getTypeName() { - return ObjectInspectorUtils.getClassShortName(primitiveClass.getName()); + return ObjectInspectorUtils.getClassShortName(primitiveClass); } } Index: src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorUtils.java =================================================================== --- src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorUtils.java (revision 711472) +++ src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorUtils.java (working copy) @@ -63,34 +63,32 @@ return primitiveClass; } + public static final Map, String> classToTypeName = new HashMap, String>(); + static { + classToTypeName.put(Boolean.class, org.apache.hadoop.hive.serde.Constants.BOOLEAN_TYPE_NAME); + classToTypeName.put(Byte.class, org.apache.hadoop.hive.serde.Constants.TINYINT_TYPE_NAME); + classToTypeName.put(Short.class, org.apache.hadoop.hive.serde.Constants.SMALLINT_TYPE_NAME); + classToTypeName.put(Integer.class, org.apache.hadoop.hive.serde.Constants.INT_TYPE_NAME); + classToTypeName.put(Long.class, org.apache.hadoop.hive.serde.Constants.BIGINT_TYPE_NAME); + classToTypeName.put(Float.class, org.apache.hadoop.hive.serde.Constants.FLOAT_TYPE_NAME); + classToTypeName.put(Double.class, org.apache.hadoop.hive.serde.Constants.DOUBLE_TYPE_NAME); + classToTypeName.put(String.class, org.apache.hadoop.hive.serde.Constants.STRING_TYPE_NAME); + classToTypeName.put(java.sql.Date.class, org.apache.hadoop.hive.serde.Constants.DATE_TYPE_NAME); + } /** * Get the short name for the types */ - public static String getClassShortName(String className) { - String result = className; - - if (result.equals(String.class.getName())) { - result = org.apache.hadoop.hive.serde.Constants.STRING_TYPE_NAME; - } else if (result.equals(Integer.class.getName())) { - result = org.apache.hadoop.hive.serde.Constants.INT_TYPE_NAME; - } else if (result.equals(Float.class.getName())) { - result = org.apache.hadoop.hive.serde.Constants.FLOAT_TYPE_NAME; - } else if (result.equals(Double.class.getName())) { - result = org.apache.hadoop.hive.serde.Constants.DOUBLE_TYPE_NAME; - } else if (result.equals(Long.class.getName())) { - result = org.apache.hadoop.hive.serde.Constants.BIGINT_TYPE_NAME; - } else if (result.equals(java.sql.Date.class.getName())) { - result = org.apache.hadoop.hive.serde.Constants.DATE_TYPE_NAME; - } else { - LOG.warn("unsupported class: " + className); + public static String getClassShortName(Class classObject) { + String result = classToTypeName.get(classObject); + if (result == null) { + result = classObject.getName(); + LOG.warn("unsupported class: " + result); + // Remove prefix + String prefix = "java.lang."; + if (result.startsWith(prefix)) { + result = result.substring(prefix.length()); + } } - - // Remove prefix - String prefix = "java.lang."; - if (result.startsWith(prefix)) { - result = result.substring(prefix.length()); - } - return result; } @@ -251,6 +249,15 @@ return fields.get(i); } } + // For backward compatibility: fieldNames can also be integer Strings. + try { + int i = Integer.parseInt(fieldName); + if (i>=0 && i= sortOrder.length() || sortOrder.charAt(topLevelStructFieldID) != '-'); + } else { + writeRawBytes(nonNullByte, 0, 1); + // If the struct is null and level > 1, DynamicSerDe will call writeNull(); + } + } + + public void writeStructEnd() throws TException { + stackLevel --; + } + + public void writeFieldBegin(TField field) throws TException { + } + + public void writeFieldEnd() throws TException { + if (stackLevel == 1) { + topLevelStructFieldID ++; + ascending = (topLevelStructFieldID >= sortOrder.length() || sortOrder.charAt(topLevelStructFieldID) != '-'); + } + } + + public void writeFieldStop() { + } + + public void writeMapBegin(TMap map) throws TException { + stackLevel++; + if (map == null) { + writeRawBytes(nonNullByte, 0, 1); + } else { + writeI32(map.size); + } + } + + public void writeMapEnd() throws TException { + stackLevel --; + } + + public void writeListBegin(TList list) throws TException { + stackLevel++; + if (list == null) { + writeRawBytes(nonNullByte, 0, 1); + } else { + writeI32(list.size); + } + } + + public void writeListEnd() throws TException { + stackLevel --; + } + + public void writeSetBegin(TSet set) throws TException { + stackLevel++; + if (set == null) { + writeRawBytes(nonNullByte, 0, 1); + } else { + writeI32(set.size); + } + } + + public void writeSetEnd() throws TException { + stackLevel --; + } + + byte[] rawBytesBuffer; + // This method takes care of bit-flipping for descending order + // Declare this method as final for performance reasons + final private void writeRawBytes(byte[] bytes, int begin, int length) throws TException { + if (ascending) { + trans_.write(bytes, begin, length); + } else { + // For fields in descending order, do a bit flip first. + if (rawBytesBuffer == null || rawBytesBuffer.length < bytes.length) { + rawBytesBuffer = new byte[bytes.length]; + } + for(int i=begin; i> 8) ^ 0x80)); + i16out[1] = (byte)(0xff & (i16)); + writeRawBytes(nonNullByte, 0, 1); + writeRawBytes(i16out, 0, 2); + } + + private byte[] i32out = new byte[4]; + public void writeI32(int i32) throws TException { + i32out[0] = (byte)(0xff & ((i32 >> 24) ^ 0x80)); + i32out[1] = (byte)(0xff & (i32 >> 16)); + i32out[2] = (byte)(0xff & (i32 >> 8)); + i32out[3] = (byte)(0xff & (i32)); + writeRawBytes(nonNullByte, 0, 1); + writeRawBytes(i32out, 0, 4); + } + + private byte[] i64out = new byte[8]; + public void writeI64(long i64) throws TException { + i64out[0] = (byte)(0xff & ((i64 >> 56) ^ 0x80)); + i64out[1] = (byte)(0xff & (i64 >> 48)); + i64out[2] = (byte)(0xff & (i64 >> 40)); + i64out[3] = (byte)(0xff & (i64 >> 32)); + i64out[4] = (byte)(0xff & (i64 >> 24)); + i64out[5] = (byte)(0xff & (i64 >> 16)); + i64out[6] = (byte)(0xff & (i64 >> 8)); + i64out[7] = (byte)(0xff & (i64)); + writeRawBytes(nonNullByte, 0, 1); + writeRawBytes(i64out, 0, 8); + } + + public void writeDouble(double dub) throws TException { + long i64 = Double.doubleToLongBits(dub); + if ((i64 & (1L << 63)) != 0) { + // negative numbers, flip all bits + i64out[0] = (byte)(0xff & ((i64 >> 56) ^ 0xff)); + i64out[1] = (byte)(0xff & ((i64 >> 48) ^ 0xff)); + i64out[2] = (byte)(0xff & ((i64 >> 40) ^ 0xff)); + i64out[3] = (byte)(0xff & ((i64 >> 32) ^ 0xff)); + i64out[4] = (byte)(0xff & ((i64 >> 24) ^ 0xff)); + i64out[5] = (byte)(0xff & ((i64 >> 16) ^ 0xff)); + i64out[6] = (byte)(0xff & ((i64 >> 8) ^ 0xff)); + i64out[7] = (byte)(0xff & ((i64) ^ 0xff)); + } else { + // positive numbers, flip just the first bit + i64out[0] = (byte)(0xff & ((i64 >> 56) ^ 0x80)); + i64out[1] = (byte)(0xff & (i64 >> 48)); + i64out[2] = (byte)(0xff & (i64 >> 40)); + i64out[3] = (byte)(0xff & (i64 >> 32)); + i64out[4] = (byte)(0xff & (i64 >> 24)); + i64out[5] = (byte)(0xff & (i64 >> 16)); + i64out[6] = (byte)(0xff & (i64 >> 8)); + i64out[7] = (byte)(0xff & (i64)); + } + writeRawBytes(nonNullByte, 0, 1); + writeRawBytes(i64out, 0, 8); + } + + final protected byte[] nullByte = new byte[] {0}; + final protected byte[] nonNullByte = new byte[] {1}; + public void writeString(String str) throws TException { + byte[] dat; + try { + dat = str.getBytes("UTF-8"); + } catch (UnsupportedEncodingException uex) { + throw new TException("JVM DOES NOT SUPPORT UTF-8: " + uex.getMessage()); + } + for(int i=0; i= sortOrder.length() || sortOrder.charAt(topLevelStructFieldID) != '-'); + } else { + // is this a null? + // only read the is-null byte for level > 1 because the top-level struct can never be null. + if (readIsNull()) return null; + } + return tstruct; + } + + public void readStructEnd() throws TException { + stackLevel --; + } + + TField f = new TField(); + public TField readFieldBegin() throws TException { + // slight hack to communicate to DynamicSerDe that the field ids are not being set but things are ordered. + f.type = -1; + return f; + } + + public void readFieldEnd() throws TException { + if (stackLevel == 1) { + topLevelStructFieldID ++; + ascending = (topLevelStructFieldID >= sortOrder.length() || sortOrder.charAt(topLevelStructFieldID) != '-'); + } + } + + private TMap tmap = new TMap(); + /** + * This method always return the same instance of TMap to avoid creating new instances. + * It is the responsibility of the caller to read the value before calling this method again. + */ + public TMap readMapBegin() throws TException { + stackLevel ++; + tmap.size = readI32(); + if (tmap.size == 0 && lastPrimitiveWasNull()) { + return null; + } + return tmap; + } + + public void readMapEnd() throws TException { + stackLevel --; + } + + private TList tlist = new TList(); + /** + * This method always return the same instance of TList to avoid creating new instances. + * It is the responsibility of the caller to read the value before calling this method again. + */ + public TList readListBegin() throws TException { + stackLevel ++; + tlist.size = readI32(); + if (tlist.size == 0 && lastPrimitiveWasNull()) { + return null; + } + return tlist; + } + + public void readListEnd() throws TException { + stackLevel --; + } + + private TSet set = new TSet(); + /** + * This method always return the same instance of TSet to avoid creating new instances. + * It is the responsibility of the caller to read the value before calling this method again. + */ + public TSet readSetBegin() throws TException { + stackLevel ++; + set.size = readI32(); + if (set.size == 0 && lastPrimitiveWasNull()) { + return null; + } + return set; + } + + public void readSetEnd() throws TException { + stackLevel --; + } + + // This method takes care of bit-flipping for descending order + // Make this method final to improve performance. + final private int readRawAll(byte[] buf, int off, int len) throws TException { + int bytes = trans_.readAll(buf, off, len); + if (!ascending) { + for(int i=off; i nativeSerDeNames = new ArrayList(); + static { + nativeSerDeNames.add(org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe.class.getName()); + nativeSerDeNames.add(org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe.class.getName()); + nativeSerDeNames.add(org.apache.hadoop.hive.serde.thrift.columnsetSerDe.class.getName()); + } + public static boolean isNativeSerDe(String serde) { + return nativeSerDeNames.contains(serde); + } + + private static boolean initCoreSerDes = registerCoreSerDes(); protected static boolean registerCoreSerDes() { Index: src/contrib/hive/serde/src/gen-java/org/apache/hadoop/hive/serde/Constants.java =================================================================== --- src/contrib/hive/serde/src/gen-java/org/apache/hadoop/hive/serde/Constants.java (revision 711472) +++ src/contrib/hive/serde/src/gen-java/org/apache/hadoop/hive/serde/Constants.java (working copy) @@ -25,6 +25,8 @@ public static final String SERIALIZATION_NULL_FORMAT = "serialization.null.format"; + public static final String SERIALIZATION_SORT_ORDER = "serialization.sort.order"; + public static final String FIELD_DELIM = "field.delim"; public static final String COLLECTION_DELIM = "colelction.delim"; @@ -33,8 +35,12 @@ public static final String MAPKEY_DELIM = "mapkey.delim"; + public static final String BOOLEAN_TYPE_NAME = "boolean"; + public static final String TINYINT_TYPE_NAME = "tinyint"; + public static final String SMALLINT_TYPE_NAME = "smallint"; + public static final String INT_TYPE_NAME = "int"; public static final String BIGINT_TYPE_NAME = "bigint"; @@ -57,7 +63,9 @@ public static final Set PrimitiveTypes = new HashSet(); static { + PrimitiveTypes.add("boolean"); PrimitiveTypes.add("tinyint"); + PrimitiveTypes.add("smallint"); PrimitiveTypes.add("int"); PrimitiveTypes.add("bigint"); PrimitiveTypes.add("float"); Index: src/contrib/hive/serde/src/gen-php/serde_constants.php =================================================================== --- src/contrib/hive/serde/src/gen-php/serde_constants.php (revision 711472) +++ src/contrib/hive/serde/src/gen-php/serde_constants.php (working copy) @@ -16,6 +16,10 @@ $GLOBALS['serde_CONSTANTS']['SERIALIZATION_DDL'] = 'serialization.ddl'; +$GLOBALS['serde_CONSTANTS']['SERIALIZATION_NULL_FORMAT'] = 'serialization.null.format'; + +$GLOBALS['serde_CONSTANTS']['SERIALIZATION_SORT_ORDER'] = 'serialization.sort.order'; + $GLOBALS['serde_CONSTANTS']['FIELD_DELIM'] = 'field.delim'; $GLOBALS['serde_CONSTANTS']['COLLECTION_DELIM'] = 'colelction.delim'; @@ -24,8 +28,12 @@ $GLOBALS['serde_CONSTANTS']['MAPKEY_DELIM'] = 'mapkey.delim'; +$GLOBALS['serde_CONSTANTS']['BOOLEAN_TYPE_NAME'] = 'boolean'; + $GLOBALS['serde_CONSTANTS']['TINYINT_TYPE_NAME'] = 'tinyint'; +$GLOBALS['serde_CONSTANTS']['SMALLINT_TYPE_NAME'] = 'smallint'; + $GLOBALS['serde_CONSTANTS']['INT_TYPE_NAME'] = 'int'; $GLOBALS['serde_CONSTANTS']['BIGINT_TYPE_NAME'] = 'bigint'; @@ -47,7 +55,9 @@ $GLOBALS['serde_CONSTANTS']['MAP_TYPE_NAME'] = 'map'; $GLOBALS['serde_CONSTANTS']['PrimitiveTypes'] = array( + 'boolean' => true, 'tinyint' => true, + 'smallint' => true, 'int' => true, 'bigint' => true, 'float' => true, Index: src/contrib/hive/serde/README =================================================================== --- src/contrib/hive/serde/README (revision 0) +++ src/contrib/hive/serde/README (revision 0) @@ -0,0 +1,115 @@ +What is SerDe +----------- +SerDe is a short name for Serializer and Deserializer. +Hive uses SerDe (and FileFormat) to read from/write to tables. + +* HDFS files --(InputFileFormat)--> --(Deserializer)--> Row object +* Row object --(Serializer)--> --(OutputFileFormat)--> HDFS files + +Note that the "key" part is ignored when reading, and is always a constant when +writing. Basically the row object is only stored into the "value". + +One principle of Hive is that Hive does not own the HDFS file format - Users +should be able to directly read the HDFS files in the Hive tables using other +tools, or use other tools to directly write to HDFS files that can be read by +Hive through "CREATE EXTERNAL TABLE", or can be loaded into Hive through "LOAD +DATA INPATH" which just move the file into Hive table directory. + + +Note that org.apache.hadoop.hive.serde is the deprecated old serde library. +Please look at org.apache.hadoop.hive.serde2 for the latest version. + + +Existing FileFormats and SerDe classes +------------------------ +Hive currently use these FileFormats to read/write to files: + +* TextInputFormat/NoKeyTextOutputFormat + These 2 classes read/write data in plain text file format. + +* SequenceFileInputFormat/SequenceFileOutputFormat + These 2 classes read/write data in hadoop SequenceFile format. + +Hive currently use these SerDe classes to serialize and deserialize data: + +* MetadataTypedColumnsetSerDe + This serde is used to read/write delimited records like CSV, tab-separated + control-A separated records (sorry, quote is not supported yet.) + +* ThriftSerDe + This serde is used to read/write thrift serialized objects. The class file + for the Thrift object must be loaded first. + +* DynamicSerDe + This serde also read/write thrift serialized objects, but it understands thrift + DDL so the schema of the object can be provided at runtime. Also it supports + a lot of different protocols, including TBinaryProtocol, TJSONProtocol, + TCTLSeparatedProtocol (which writes data in delimited records). + + + +How to load data into Hive +------------------------ +In order to load data into Hive, we need to tell Hive the format of the data +through "CREATE TABLE" statement: + +* FileFormat: the data has to be in Text or SequenceFile. +* Format of the row: + * If the data is in delimited format, use MetadataTypedColumnsetSerDe + * If the data is in delimited format and has more than 1 levels of delimitor, + use DynamicSerDe with TCTLSeparatedProtocol + * If the data is a serialized thrift object, use ThriftSerDe + +The steps to load the data: +1 Create a table: + + CREATE TABLE t (foo STRING, bar STRING) + ROW FORMAT DELIMITED + FIELDS TERMINATED BY '\t' + STORED AS TEXTFILE; + + CREATE TABLE t2 (foo STRING, bar ARRAY) + ROW FORMAT DELIMITED + FIELDS TERMINATED BY '\t' + COLLECTION ITEMS TERMINATED BY ',' + STORED AS TEXTFILE; + + CREATE TABLE t3 (foo STRING, bar MAP) + ROW FORMAT DELIMITED + FIELDS TERMINATED BY '\t' + COLLECTION ITEMS TERMINATED BY ',' + MAP KEYS TERMINATED BY ':' + STORED AS TEXTFILE; + + CREATE TABLE t4 (foo STRING, bar MAP) + ROW FORMAT SERIALIZER 'org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe' + WITH SERDEPROPERTIES ('columns'='foo,bar','SERIALIZATION.FORMAT'='9'); + + (RegexDeserializer is not done yet) + CREATE TABLE t5 (foo STRING, bar STRING) + ROW FORMAT SERIALIZER 'org.apache.hadoop.hive.serde2.RegexDeserializer' + WITH SERDEPROPERTIES ('regex'='([a-z]*) *([a-z]*)'); + +2 Load the data: + LOAD DATA LOCAL INPATH '../examples/files/kv1.txt' OVERWRITE INTO TABLE t; + + + +How to read data from Hive tables +------------------------ +In order to read data from Hive tables, we need to know the same 2 things: +* File Format +* Row Format + +Then we just need to directly open the HDFS file and read the data. + + +How to write your own SerDe +------------------------ + +In most cases, users want to write a Deserializer instead of a SerDe. +For example, the RegexDeserializer will deserialize the data using the +configuration parameter 'regex', and possibly a list of column names (see +serde2.MetadataTypedColumnsetSerDe). + +Please see serde2/Deserializer.java for details. Index: src/contrib/hive/serde/if/serde.thrift =================================================================== --- src/contrib/hive/serde/if/serde.thrift (revision 711472) +++ src/contrib/hive/serde/if/serde.thrift (working copy) @@ -9,6 +9,7 @@ const string SERIALIZATION_FORMAT = "serialization.format" const string SERIALIZATION_DDL = "serialization.ddl" const string SERIALIZATION_NULL_FORMAT = "serialization.null.format" +const string SERIALIZATION_SORT_ORDER = "serialization.sort.order" const string FIELD_DELIM = "field.delim" const string COLLECTION_DELIM = "colelction.delim" @@ -18,11 +19,13 @@ typedef string PrimitiveType typedef string CollectionType +const string BOOLEAN_TYPE_NAME = "boolean"; const string TINYINT_TYPE_NAME = "tinyint"; +const string SMALLINT_TYPE_NAME = "smallint"; const string INT_TYPE_NAME = "int"; const string BIGINT_TYPE_NAME = "bigint"; const string FLOAT_TYPE_NAME = "float"; -const string DOUBLE_TYPE_NAME = "double"; +const string DOUBLE_TYPE_NAME = "double"; const string STRING_TYPE_NAME = "string"; const string DATE_TYPE_NAME = "date"; const string DATETIME_TYPE_NAME = "datetime"; @@ -31,7 +34,7 @@ const string LIST_TYPE_NAME = "array"; const string MAP_TYPE_NAME = "map"; -const set PrimitiveTypes = [ TINYINT_TYPE_NAME INT_TYPE_NAME BIGINT_TYPE_NAME FLOAT_TYPE_NAME DOUBLE_TYPE_NAME STRING_TYPE_NAME DATE_TYPE_NAME DATETIME_TYPE_NAME TIMESTAMP_TYPE_NAME ], +const set PrimitiveTypes = [ BOOLEAN_TYPE_NAME TINYINT_TYPE_NAME SMALLINT_TYPE_NAME INT_TYPE_NAME BIGINT_TYPE_NAME FLOAT_TYPE_NAME DOUBLE_TYPE_NAME STRING_TYPE_NAME DATE_TYPE_NAME DATETIME_TYPE_NAME TIMESTAMP_TYPE_NAME ], const set CollectionTypes = [ LIST_TYPE_NAME MAP_TYPE_NAME ], Index: src/contrib/hive/metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java =================================================================== --- src/contrib/hive/metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java (revision 711472) +++ src/contrib/hive/metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java (working copy) @@ -28,6 +28,8 @@ import org.apache.hadoop.hive.serde.Constants; import org.apache.hadoop.hive.metastore.api.Database; import org.apache.hadoop.hive.metastore.api.FieldSchema; +import org.apache.hadoop.hive.metastore.api.InvalidObjectException; +import org.apache.hadoop.hive.metastore.api.InvalidOperationException; import org.apache.hadoop.hive.metastore.api.Order; import org.apache.hadoop.hive.metastore.api.Partition; import org.apache.hadoop.hive.metastore.api.SerDeInfo; @@ -350,6 +352,81 @@ } } + public void testAlterTable() throws Exception { + try { + String dbName = "alterdb"; + String invTblName = "alter-tbl"; + String tblName = "altertbl"; + + client.dropTable(dbName, tblName); + client.dropDatabase(dbName); + boolean ret = client.createDatabase(dbName, "strange_loc"); + assertTrue("Unable to create the databse " + dbName, ret); + + ArrayList invCols = new ArrayList(2); + invCols.add(new FieldSchema("n-ame", Constants.STRING_TYPE_NAME, "")); + invCols.add(new FieldSchema("in.come", Constants.INT_TYPE_NAME, "")); + + Table tbl = new Table(); + tbl.setDbName(dbName); + tbl.setTableName(invTblName); + StorageDescriptor sd = new StorageDescriptor(); + tbl.setSd(sd); + sd.setCols(invCols); + sd.setCompressed(false); + sd.setNumBuckets(1); + sd.setParameters(new HashMap()); + sd.getParameters().put("test_param_1", "Use this for comments etc"); + sd.setBucketCols(new ArrayList(2)); + sd.getBucketCols().add("name"); + sd.setSerdeInfo(new SerDeInfo()); + sd.getSerdeInfo().setName(tbl.getTableName()); + sd.getSerdeInfo().setParameters(new HashMap()); + sd.getSerdeInfo().getParameters().put(org.apache.hadoop.hive.serde.Constants.SERIALIZATION_FORMAT, "1"); + boolean failed = false; + try { + client.createTable(tbl); + } catch (InvalidObjectException ex) { + failed = true; + } + if(!failed) { + assertTrue("Able to create table with invalid name: " + invTblName, false); + } + ArrayList cols = new ArrayList(2); + cols.add(new FieldSchema("name", Constants.STRING_TYPE_NAME, "")); + cols.add(new FieldSchema("income", Constants.INT_TYPE_NAME, "")); + + // create a valid table + tbl.setTableName(tblName); + tbl.getSd().setCols(cols); + client.createTable(tbl); + + // now try to invalid alter table + Table tbl2 = client.getTable(dbName, tblName); + failed = false; + try { + tbl2.setTableName(invTblName); + tbl2.getSd().setCols(invCols); + client.alter_table(dbName, tblName, tbl2); + } catch (InvalidOperationException ex) { + failed = true; + } + if(!failed) { + assertTrue("Able to rename table with invalid name: " + invTblName, false); + } + // try a valid alter table + tbl2.setTableName(tblName); + tbl2.getSd().setCols(cols); + tbl2.getSd().setNumBuckets(32); + client.alter_table(dbName, tblName, tbl2); + Table tbl3 = client.getTable(dbName, tblName); + assertEquals("Alter table didn't succeed. Num buckets ", tbl2.getSd().getNumBuckets(), tbl3.getSd().getNumBuckets()); + } catch (Exception e) { + System.err.println(StringUtils.stringifyException(e)); + System.err.println("testSimpleTable() failed."); + throw e; + } + } public void testComplexTable() throws Exception { String dbName = "compdb"; Index: src/contrib/hive/metastore/src/test/org/apache/hadoop/hive/metastore/TestPartitions.java =================================================================== --- src/contrib/hive/metastore/src/test/org/apache/hadoop/hive/metastore/TestPartitions.java (revision 711472) +++ src/contrib/hive/metastore/src/test/org/apache/hadoop/hive/metastore/TestPartitions.java (working copy) @@ -60,8 +60,8 @@ fileSys_.mkdirs(part2); List partitions = bar1.getPartitions(); assertTrue(partitions.size() == 2); - assertTrue(partitions.get(0).equals("ds=2008-01-01")); - assertTrue(partitions.get(1).equals("ds=2008-01-02")); + assertTrue(partitions.contains("ds=2008-01-01")); + assertTrue(partitions.contains("ds=2008-01-02")); cleanup(); } catch(MetaException e) { e.printStackTrace(); Index: src/contrib/hive/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java =================================================================== --- src/contrib/hive/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java (revision 711472) +++ src/contrib/hive/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java (working copy) @@ -271,8 +271,10 @@ this.incrementCounter("create_table"); logStartFunction("create_table: db=" + tbl.getDbName() + " tbl=" + tbl.getTableName()); boolean success = false; - if(!MetaStoreUtils.validateName(tbl.getTableName())) { - throw new InvalidObjectException(tbl.getTableName() + " is not a valid object name"); + if(!MetaStoreUtils.validateName(tbl.getTableName()) || + !MetaStoreUtils.validateColNames(tbl.getSd().getCols()) || + (tbl.getPartitionKeys() != null && !MetaStoreUtils.validateColNames(tbl.getPartitionKeys()))) { + throw new InvalidObjectException(tbl.getTableName() + " is not a valid object name"); } try { getMS().openTransaction(); @@ -540,11 +542,15 @@ logStartFunction("getVersion"); return "3.0"; } - + public void alter_table(String dbname, String name, Table newTable) throws InvalidOperationException, MetaException { this.incrementCounter("alter_table"); logStartFunction("truncate_table: db=" + dbname + " tbl=" + name + " newtbl=" + newTable.getTableName()); + if(!MetaStoreUtils.validateName(newTable.getTableName()) || + !MetaStoreUtils.validateColNames(newTable.getSd().getCols())) { + throw new InvalidOperationException(newTable.getTableName() + " is not a valid object name"); + } try { getMS().alterTable(dbname, name, newTable); } catch (InvalidObjectException e) { Index: src/contrib/hive/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java =================================================================== --- src/contrib/hive/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java (revision 711472) +++ src/contrib/hive/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java (working copy) @@ -231,8 +231,8 @@ /** * validateName * - * Checks the name conforms to our standars which are: "[a-zA-z-_0-9]+". - * checks this is just characters and numbers and _ and . and - + * Checks the name conforms to our standars which are: "[a-zA-z_0-9]+". + * checks this is just characters and numbers and _ * * @param tableName the name to validate * @return none @@ -246,6 +246,14 @@ } return false; } + + static public boolean validateColNames(List cols) { + for (FieldSchema fieldSchema : cols) { + if(!validateName(fieldSchema.getName())) + return false; + } + return true; + } /** * Change from old to new format properties of a schema file @@ -395,7 +403,9 @@ static HashMap typeToThriftTypeMap; static { typeToThriftTypeMap = new HashMap(); + typeToThriftTypeMap.put(org.apache.hadoop.hive.serde.Constants.BOOLEAN_TYPE_NAME, "bool"); typeToThriftTypeMap.put(org.apache.hadoop.hive.serde.Constants.TINYINT_TYPE_NAME, "byte"); + typeToThriftTypeMap.put(org.apache.hadoop.hive.serde.Constants.SMALLINT_TYPE_NAME, "i16"); typeToThriftTypeMap.put(org.apache.hadoop.hive.serde.Constants.INT_TYPE_NAME, "i32"); typeToThriftTypeMap.put(org.apache.hadoop.hive.serde.Constants.BIGINT_TYPE_NAME, "i64"); typeToThriftTypeMap.put(org.apache.hadoop.hive.serde.Constants.DOUBLE_TYPE_NAME, "double"); @@ -446,7 +456,7 @@ ddl.append(col.getName()); } ddl.append("}"); - LOG.warn("DDL: " + ddl); + LOG.info("DDL: " + ddl); return ddl.toString(); } public static Properties getSchema(org.apache.hadoop.hive.metastore.api.Table tbl) { @@ -538,7 +548,7 @@ * @throws SerDeException * @throws MetaException */ - static List getFieldsFromDeserializer(String tableName, Deserializer deserializer) throws SerDeException, MetaException { + public static List getFieldsFromDeserializer(String tableName, Deserializer deserializer) throws SerDeException, MetaException { ObjectInspector oi = deserializer.getObjectInspector(); String [] names = tableName.split("\\."); String last_name = names[names.length-1]; @@ -572,4 +582,5 @@ } return str_fields; } + } Index: src/contrib/hive/conf/hive-default.xml =================================================================== --- src/contrib/hive/conf/hive-default.xml (revision 711472) +++ src/contrib/hive/conf/hive-default.xml (working copy) @@ -84,4 +84,22 @@ Name of the class that implements org.apache.hadoop.hive.metastore.rawstore interface. This class is used to store and retrieval of raw metadata objects such as table, database + + hive.default.fileformat + TextFile + Default file format for CREATE TABLE statement. Options are TextFile and SequenceFile. Users can explicitly say CREATE TABLE ... STORED AS <TEXTFILE|SEQUENCEFILE> to override + + + + hive.map.aggr + false + Whether to use map-side aggregation in Hive Group By queries + + + + hive.join.emit.interval + 1000 + How many rows in the right-most join operand Hive should buffer before emitting the join result. + + Index: src/contrib/hive/cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java =================================================================== --- src/contrib/hive/cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java (revision 711472) +++ src/contrib/hive/cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java (working copy) @@ -31,6 +31,7 @@ import org.apache.hadoop.hive.ql.Driver; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.commons.lang.StringUtils; import org.apache.hadoop.hive.ql.session.SessionState.LogHelper; public class CliDriver { @@ -41,7 +42,18 @@ public static SetProcessor sp; public static Driver qp; public static FsShell dfs; + public static Log LOG = LogFactory.getLog("CliDriver"); + /** + * delay console initialization until session has been initialized + */ + public static LogHelper console; + public static LogHelper getConsole() { + if(console == null) + console = new LogHelper(LOG); + return (console); + } + public CliDriver(CliSessionState ss) { SessionState.start(ss); sp = new SetProcessor(); @@ -49,23 +61,28 @@ } public static int processCmd(String cmd) { + + SessionState ss = SessionState.get(); + LogHelper console = getConsole(); + String[] tokens = cmd.split("\\s+"); String cmd_1 = cmd.substring(tokens[0].length()); int ret = 0; - if(tokens[0].equals("set")) { + if(tokens[0].toLowerCase().equals("set")) { + ret = sp.run(cmd_1); - } else if (cmd.equals("quit") || cmd.equals("exit")) { + + } else if (cmd.toLowerCase().equals("quit") || cmd.toLowerCase().equals("exit")) { + // if we have come this far - either the previous commands // are all successful or this is command line. in either case // this counts as a successful run System.exit(0); + } else if (cmd.startsWith("!")) { - SessionState ss = SessionState.get(); + String shell_cmd = cmd.substring(1); - if (shell_cmd.endsWith(";")) { - shell_cmd = shell_cmd.substring(0, shell_cmd.length()-1); - } //shell_cmd = "/bin/bash -c \'" + shell_cmd + "\'"; try { @@ -76,48 +93,102 @@ outPrinter.start(); errPrinter.start(); - int exitVal = executor.waitFor(); - if (exitVal != 0) { - ss.err.write((new String("Command failed with exit code = " + exitVal)).getBytes()); + ret = executor.waitFor(); + if (ret != 0) { + console.printError("Command failed with exit code = " + ret); } } catch (Exception e) { - e.printStackTrace(); + console.printError("Exception raised from Shell command " + e.getLocalizedMessage(), + org.apache.hadoop.util.StringUtils.stringifyException(e)); + ret = 1; } - } else if (cmd.startsWith("dfs")) { + + } else if (tokens[0].toLowerCase().equals("dfs")) { + // dfs shell commands - SessionState ss = SessionState.get(); if(dfs == null) dfs = new FsShell(ss.getConf()); - String hadoopCmd = cmd.replaceFirst("dfs\\s+", ""); - hadoopCmd = hadoopCmd.trim(); - if (hadoopCmd.endsWith(";")) { - hadoopCmd = hadoopCmd.substring(0, hadoopCmd.length()-1); - } - String[] args = hadoopCmd.split("\\s+"); + + String [] alt_tokens = new String [tokens.length-1]; + System.arraycopy(tokens, 1, alt_tokens, 0, tokens.length-1); + tokens = alt_tokens; + try { PrintStream oldOut = System.out; System.setOut(ss.out); - int val = dfs.run(args); + ret = dfs.run(tokens); System.setOut(oldOut); - if (val != 0) { - ss.err.write((new String("Command failed with exit code = " + val)).getBytes()); + if (ret != 0) { + console.printError("Command failed with exit code = " + ret); } } catch (Exception e) { - ss.err.println("Exception raised from DFSShell.run " + e.getLocalizedMessage()); + console.printError("Exception raised from DFSShell.run " + e.getLocalizedMessage(), + org.apache.hadoop.util.StringUtils.stringifyException(e)); + ret = 1; } + + } else if (tokens[0].toLowerCase().equals("list")) { + + SessionState.ResourceType t; + if(tokens.length < 2 || (t = SessionState.find_resource_type(tokens[1])) == null) { + console.printError("Usage: list [" + + StringUtils.join(SessionState.ResourceType.values(),"|") + + "] [ []*]" ); + ret = 1; + } else { + List filter = null; + if(tokens.length >=3) { + System.arraycopy(tokens, 2, tokens, 0, tokens.length-2); + filter = Arrays.asList(tokens); + } + Set s = ss.list_resource(t, filter); + if(s != null && !s.isEmpty()) + ss.out.println(StringUtils.join(s, "\n")); + } + + } else if (tokens[0].toLowerCase().equals("add")) { + + SessionState.ResourceType t; + if(tokens.length < 3 || (t = SessionState.find_resource_type(tokens[1])) == null) { + console.printError("Usage: add [" + + StringUtils.join(SessionState.ResourceType.values(),"|") + + "] []*"); + ret = 1; + } else { + for(int i = 2; i]"); + ret = 1; + } else if (tokens.length >= 3) { + for(int i = 2; i res = new Vector(); while (qp.getResults(res)) { for (String r:res) { - SessionState ss = SessionState.get(); - PrintStream out = ss.out; out.println(r); } res.clear(); } - + int cret = qp.close(); if (ret == 0) { ret = cret; @@ -223,8 +294,7 @@ String historyFile = System.getProperty("user.home") + File.separator + HISTORYFILE; reader.setHistory(new History(new File(historyFile))); int ret = 0; - Log LOG = LogFactory.getLog("CliDriver"); - LogHelper console = new LogHelper(LOG); + String prefix = ""; String curPrompt = prompt; while ((line = reader.readLine(curPrompt+"> ")) != null) { @@ -242,10 +312,11 @@ long end = System.currentTimeMillis(); if (end > start) { double timeTaken = (double)(end-start)/1000.0; - console.printInfo("Time taken: " + timeTaken + " seconds", null); + getConsole().printInfo("Time taken: " + timeTaken + " seconds", null); } } System.exit(ret); } + } Index: src/contrib/hive/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java =================================================================== --- src/contrib/hive/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (revision 711472) +++ src/contrib/hive/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (working copy) @@ -81,8 +81,14 @@ HIVETABLENAME("hive.table.name", ""), HIVEPARTITIONNAME("hive.partition.name", ""), HIVEPARTITIONPRUNER("hive.partition.pruning", "nonstrict"), - HIVEALIAS("hive.alias", ""); + HIVEALIAS("hive.alias", ""), + HIVEMAPSIDEAGGREGATE("hive.map.aggr", "false"), + HIVEJOINEMITINTERVAL("hive.join.emit.interval", 1000), + // Default file format for CREATE TABLE statement + // Options: TextFile, SequenceFile + HIVEDEFAULTFILEFORMAT("hive.default.fileformat", "TextFile"); + public final String varname; public final String defaultVal; public final int defaultIntVal; Index: src/contrib/hive/ql/src/test/results/clientnegative/invalid_create_tbl1.q.out =================================================================== --- src/contrib/hive/ql/src/test/results/clientnegative/invalid_create_tbl1.q.out (revision 0) +++ src/contrib/hive/ql/src/test/results/clientnegative/invalid_create_tbl1.q.out (revision 0) @@ -0,0 +1,2 @@ +FAILED: Error in metadata: org.apache.hadoop.hive.ql.metadata.HiveException: Partition collumn name aint conflicts with table columns. +FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask Index: src/contrib/hive/ql/src/test/results/clientnegative/notable_alias3.q.out =================================================================== --- src/contrib/hive/ql/src/test/results/clientnegative/notable_alias3.q.out (revision 0) +++ src/contrib/hive/ql/src/test/results/clientnegative/notable_alias3.q.out (revision 0) @@ -0,0 +1 @@ +FAILED: Error in semantic analysis: line 2:44 Expression Not In Group By Key key Index: src/contrib/hive/ql/src/test/results/clientnegative/notable_alias4.q.out =================================================================== --- src/contrib/hive/ql/src/test/results/clientnegative/notable_alias4.q.out (revision 0) +++ src/contrib/hive/ql/src/test/results/clientnegative/notable_alias4.q.out (revision 0) @@ -0,0 +1 @@ +FAILED: Error in semantic analysis: Column key Found in more than One Tables/Subqueries Index: src/contrib/hive/ql/src/test/results/clientnegative/bad_sample_clause.q.out =================================================================== --- src/contrib/hive/ql/src/test/results/clientnegative/bad_sample_clause.q.out (revision 0) +++ src/contrib/hive/ql/src/test/results/clientnegative/bad_sample_clause.q.out (revision 0) @@ -0,0 +1 @@ +FAILED: Error in semantic analysis: Sampling Expression Needed for Non-Bucketed Table srcpart Index: src/contrib/hive/ql/src/test/results/clientnegative/load_wrong_fileformat.q.out =================================================================== --- src/contrib/hive/ql/src/test/results/clientnegative/load_wrong_fileformat.q.out (revision 0) +++ src/contrib/hive/ql/src/test/results/clientnegative/load_wrong_fileformat.q.out (revision 0) @@ -0,0 +1,2 @@ +Failed with exception Cannot load text files into a table stored as SequenceFile. +FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.MoveTask Index: src/contrib/hive/ql/src/test/results/clientnegative/input_testxpath4.q.out =================================================================== --- src/contrib/hive/ql/src/test/results/clientnegative/input_testxpath4.q.out (revision 0) +++ src/contrib/hive/ql/src/test/results/clientnegative/input_testxpath4.q.out (revision 0) @@ -0,0 +1 @@ +FAILED: Error in semantic analysis: line 3:42 Invalid Table Alias lintstring Index: src/contrib/hive/ql/src/test/results/clientnegative/invalid_tbl_name.q.out =================================================================== --- src/contrib/hive/ql/src/test/results/clientnegative/invalid_tbl_name.q.out (revision 0) +++ src/contrib/hive/ql/src/test/results/clientnegative/invalid_tbl_name.q.out (revision 0) @@ -0,0 +1,2 @@ +FAILED: Parse Error: line 1:20 mismatched input '-' expecting EOF + Index: src/contrib/hive/ql/src/test/results/clientpositive/inputddl8.q.out =================================================================== --- src/contrib/hive/ql/src/test/results/clientpositive/inputddl8.q.out (revision 0) +++ src/contrib/hive/ql/src/test/results/clientpositive/inputddl8.q.out (revision 0) @@ -0,0 +1,10 @@ +aint int 'from deserializer' +astring string 'from deserializer' +lint array 'from deserializer' +lstring array 'from deserializer' +lintstring array 'from deserializer' +mstringstring map 'from deserializer' +ds datetime +country string +Detailed Table Information: +Table(tableName:inputddl8,dbName:default,owner:athusoo,createTime:1225764124,lastAccessTime:0,retention:0,sd:StorageDescriptor(cols:[],location:file:/data/users/athusoo/hadoop_local_ws1/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/inputddl8,inputFormat:org.apache.hadoop.mapred.SequenceFileInputFormat,outputFormat:org.apache.hadoop.mapred.SequenceFileOutputFormat,compressed:false,numBuckets:32,serdeInfo:SerDeInfo(name:null,serializationLib:org.apache.hadoop.hive.serde2.ThriftDeserializer,parameters:{serialization.class=org.apache.hadoop.hive.serde2.thrift.test.Complex,serialization.format=com.facebook.thrift.protocol.TBinaryProtocol}),bucketCols:[aint],sortCols:[Order(col:lint,order:1)],parameters:{}),partitionKeys:[FieldSchema(name:ds,type:datetime,comment:null), FieldSchema(name:country,type:string,comment:null)],parameters:{comment=This is a thrift based table}) Index: src/contrib/hive/ql/src/test/results/clientpositive/cast1.q.out =================================================================== --- src/contrib/hive/ql/src/test/results/clientpositive/cast1.q.out (revision 711472) +++ src/contrib/hive/ql/src/test/results/clientpositive/cast1.q.out (working copy) @@ -13,7 +13,7 @@ Filter Operator predicate: expr: (key = 86) - type: Boolean + type: boolean Select Operator expressions: expr: (3 + 2) @@ -27,7 +27,7 @@ expr: (3 + UDFToInteger(2.0)) type: int expr: UDFToBoolean(1) - type: Boolean + type: boolean expr: UDFToInteger(true) type: int File Output Operator Index: src/contrib/hive/ql/src/test/results/clientpositive/quote1.q.out =================================================================== --- src/contrib/hive/ql/src/test/results/clientpositive/quote1.q.out (revision 0) +++ src/contrib/hive/ql/src/test/results/clientpositive/quote1.q.out (revision 0) @@ -0,0 +1,179 @@ +ABSTRACT SYNTAX TREE: + (TOK_QUERY (TOK_FROM (TOK_TABREF src)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB dest1 (TOK_PARTSPEC (TOK_PARTVAL `table` '2008-04-08')))) (TOK_SELECT (TOK_SELEXPR (TOK_COLREF src key) `partition`) (TOK_SELEXPR (TOK_COLREF src value) `from`)) (TOK_WHERE (and (>= (TOK_COLREF src key) 200) (< (TOK_COLREF src key) 300))))) + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Alias -> Map Operator Tree: + src + Filter Operator + predicate: + expr: ((key >= 200) and (key < 300)) + type: boolean + Select Operator + expressions: + expr: key + type: string + expr: value + type: string + File Output Operator + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe + name: dest1 + + Stage: Stage-0 + Move Operator + tables: + partition: + table 2008-04-08 + replace: + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe + name: dest1 + + +ABSTRACT SYNTAX TREE: + (TOK_QUERY (TOK_FROM (TOK_TABREF dest1 `table`)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_COLREF `table` `location`)) (TOK_SELEXPR (TOK_COLREF `table` `type`)) (TOK_SELEXPR (TOK_COLREF `table` `table`))) (TOK_WHERE (= (TOK_COLREF `table` `table`) '2008-04-08')))) + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Alias -> Map Operator Tree: + table + Filter Operator + predicate: + expr: (table = '2008-04-08') + type: boolean + Select Operator + expressions: + expr: location + type: string + expr: type + type: string + expr: table + type: string + File Output Operator + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + + Stage: Stage-0 + Fetch Operator + limit: -1 + + +238 val_238 2008-04-08 +255 val_255 2008-04-08 +278 val_278 2008-04-08 +265 val_265 2008-04-08 +273 val_273 2008-04-08 +224 val_224 2008-04-08 +213 val_213 2008-04-08 +281 val_281 2008-04-08 +277 val_277 2008-04-08 +209 val_209 2008-04-08 +252 val_252 2008-04-08 +292 val_292 2008-04-08 +219 val_219 2008-04-08 +287 val_287 2008-04-08 +237 val_237 2008-04-08 +207 val_207 2008-04-08 +208 val_208 2008-04-08 +247 val_247 2008-04-08 +266 val_266 2008-04-08 +203 val_203 2008-04-08 +205 val_205 2008-04-08 +221 val_221 2008-04-08 +280 val_280 2008-04-08 +277 val_277 2008-04-08 +208 val_208 2008-04-08 +286 val_286 2008-04-08 +239 val_239 2008-04-08 +213 val_213 2008-04-08 +216 val_216 2008-04-08 +278 val_278 2008-04-08 +289 val_289 2008-04-08 +221 val_221 2008-04-08 +275 val_275 2008-04-08 +241 val_241 2008-04-08 +284 val_284 2008-04-08 +230 val_230 2008-04-08 +260 val_260 2008-04-08 +272 val_272 2008-04-08 +217 val_217 2008-04-08 +230 val_230 2008-04-08 +208 val_208 2008-04-08 +298 val_298 2008-04-08 +230 val_230 2008-04-08 +205 val_205 2008-04-08 +288 val_288 2008-04-08 +282 val_282 2008-04-08 +282 val_282 2008-04-08 +238 val_238 2008-04-08 +277 val_277 2008-04-08 +273 val_273 2008-04-08 +224 val_224 2008-04-08 +242 val_242 2008-04-08 +272 val_272 2008-04-08 +242 val_242 2008-04-08 +226 val_226 2008-04-08 +229 val_229 2008-04-08 +233 val_233 2008-04-08 +223 val_223 2008-04-08 +218 val_218 2008-04-08 +228 val_228 2008-04-08 +209 val_209 2008-04-08 +230 val_230 2008-04-08 +296 val_296 2008-04-08 +216 val_216 2008-04-08 +274 val_274 2008-04-08 +219 val_219 2008-04-08 +239 val_239 2008-04-08 +223 val_223 2008-04-08 +256 val_256 2008-04-08 +263 val_263 2008-04-08 +288 val_288 2008-04-08 +244 val_244 2008-04-08 +202 val_202 2008-04-08 +229 val_229 2008-04-08 +280 val_280 2008-04-08 +283 val_283 2008-04-08 +235 val_235 2008-04-08 +257 val_257 2008-04-08 +258 val_258 2008-04-08 +203 val_203 2008-04-08 +262 val_262 2008-04-08 +201 val_201 2008-04-08 +217 val_217 2008-04-08 +298 val_298 2008-04-08 +291 val_291 2008-04-08 +255 val_255 2008-04-08 +200 val_200 2008-04-08 +237 val_237 2008-04-08 +248 val_248 2008-04-08 +277 val_277 2008-04-08 +230 val_230 2008-04-08 +207 val_207 2008-04-08 +249 val_249 2008-04-08 +265 val_265 2008-04-08 +214 val_214 2008-04-08 +233 val_233 2008-04-08 +256 val_256 2008-04-08 +298 val_298 2008-04-08 +285 val_285 2008-04-08 +273 val_273 2008-04-08 +281 val_281 2008-04-08 +222 val_222 2008-04-08 +200 val_200 2008-04-08 Index: src/contrib/hive/ql/src/test/results/clientpositive/notable_alias2.q.out =================================================================== --- src/contrib/hive/ql/src/test/results/clientpositive/notable_alias2.q.out (revision 0) +++ src/contrib/hive/ql/src/test/results/clientpositive/notable_alias2.q.out (revision 0) @@ -0,0 +1,144 @@ +ABSTRACT SYNTAX TREE: + (TOK_QUERY (TOK_FROM (TOK_TABREF src)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB dest1)) (TOK_SELECT (TOK_SELEXPR '1234') (TOK_SELEXPR (TOK_COLREF src key)) (TOK_SELEXPR (TOK_FUNCTION count 1))) (TOK_WHERE (< (TOK_COLREF key) 100)) (TOK_GROUPBY (TOK_COLREF src key)))) + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Alias -> Map Operator Tree: + src + Filter Operator + predicate: + expr: (key < 100) + type: boolean + Reduce Output Operator + key expressions: + expr: key + type: string + # partition fields: -1 + tag: -1 + value expressions: + expr: 1 + type: int + Reduce Operator Tree: + Group By Operator + + expr: count(VALUE.0) + keys: + expr: KEY.0 + type: string + mode: partial1 + File Output Operator + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.mapred.SequenceFileOutputFormat + name: binary_table + + Stage: Stage-2 + Map Reduce + Alias -> Map Operator Tree: + /tmp/hive-athusoo/306970709/884069827.10001 + Reduce Output Operator + key expressions: + expr: 0 + type: string + # partition fields: 1 + tag: -1 + value expressions: + expr: 1 + type: bigint + Reduce Operator Tree: + Group By Operator + + expr: count(VALUE.0) + keys: + expr: KEY.0 + type: string + mode: unknown + Select Operator + expressions: + expr: '1234' + type: string + expr: 0 + type: string + expr: 1 + type: bigint + File Output Operator + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe + name: dest1 + + Stage: Stage-0 + Move Operator + tables: + replace: + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe + name: dest1 + + +1234 0 3 +1234 10 1 +1234 11 1 +1234 12 2 +1234 15 2 +1234 17 1 +1234 18 2 +1234 19 1 +1234 2 1 +1234 20 1 +1234 24 2 +1234 26 2 +1234 27 1 +1234 28 1 +1234 30 1 +1234 33 1 +1234 34 1 +1234 35 3 +1234 37 2 +1234 4 1 +1234 41 1 +1234 42 2 +1234 43 1 +1234 44 1 +1234 47 1 +1234 5 3 +1234 51 2 +1234 53 1 +1234 54 1 +1234 57 1 +1234 58 2 +1234 64 1 +1234 65 1 +1234 66 1 +1234 67 2 +1234 69 1 +1234 70 3 +1234 72 2 +1234 74 1 +1234 76 2 +1234 77 1 +1234 78 1 +1234 8 1 +1234 80 1 +1234 82 1 +1234 83 2 +1234 84 2 +1234 85 1 +1234 86 1 +1234 87 1 +1234 9 1 +1234 90 3 +1234 92 1 +1234 95 2 +1234 96 1 +1234 97 2 +1234 98 2 Index: src/contrib/hive/ql/src/test/results/clientpositive/join2.q.out =================================================================== --- src/contrib/hive/ql/src/test/results/clientpositive/join2.q.out (revision 711472) +++ src/contrib/hive/ql/src/test/results/clientpositive/join2.q.out (working copy) @@ -44,8 +44,9 @@ 1 {VALUE.0} {VALUE.1} File Output Operator table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.mapred.SequenceFileOutputFormat + name: binary_table Stage: Stage-2 Map Reduce @@ -136,6 +137,8 @@ 0 val_0 0 val_0 0 val_0 +2 val_4 +4 val_8 5 val_10 5 val_10 5 val_10 @@ -145,6 +148,65 @@ 5 val_10 5 val_10 5 val_10 +9 val_18 +9 val_18 +10 val_20 +12 val_24 +12 val_24 +12 val_24 +12 val_24 +12 val_24 +12 val_24 +12 val_24 +12 val_24 +15 val_30 +15 val_30 +15 val_30 +15 val_30 +17 val_34 +27 val_54 +33 val_66 +35 val_70 +35 val_70 +35 val_70 +35 val_70 +35 val_70 +35 val_70 +35 val_70 +35 val_70 +35 val_70 +35 val_70 +35 val_70 +35 val_70 +35 val_70 +35 val_70 +35 val_70 +35 val_70 +35 val_70 +35 val_70 +35 val_70 +35 val_70 +35 val_70 +35 val_70 +35 val_70 +35 val_70 +35 val_70 +35 val_70 +35 val_70 +37 val_74 +37 val_74 +37 val_74 +37 val_74 +41 val_82 +42 val_84 +42 val_84 +42 val_84 +42 val_84 +42 val_84 +42 val_84 +42 val_84 +42 val_84 +43 val_86 57 val_114 58 val_116 58 val_116 @@ -190,8 +252,6 @@ 86 val_172 87 val_174 87 val_174 -9 val_18 -9 val_18 90 val_180 90 val_180 90 val_180 @@ -214,7 +274,6 @@ 98 val_196 98 val_196 98 val_196 -10 val_20 100 val_200 100 val_200 100 val_200 @@ -259,14 +318,6 @@ 119 val_238 119 val_238 119 val_238 -12 val_24 -12 val_24 -12 val_24 -12 val_24 -12 val_24 -12 val_24 -12 val_24 -12 val_24 126 val_252 128 val_256 128 val_256 @@ -315,10 +366,6 @@ 149 val_298 149 val_298 149 val_298 -15 val_30 -15 val_30 -15 val_30 -15 val_30 153 val_306 155 val_310 158 val_316 @@ -342,7 +389,6 @@ 169 val_338 169 val_338 169 val_338 -17 val_34 172 val_344 172 val_344 172 val_344 @@ -410,7 +456,6 @@ 197 val_394 197 val_394 197 val_394 -2 val_4 200 val_400 200 val_400 200 val_400 @@ -537,47 +582,3 @@ 249 val_498 249 val_498 249 val_498 -27 val_54 -33 val_66 -35 val_70 -35 val_70 -35 val_70 -35 val_70 -35 val_70 -35 val_70 -35 val_70 -35 val_70 -35 val_70 -35 val_70 -35 val_70 -35 val_70 -35 val_70 -35 val_70 -35 val_70 -35 val_70 -35 val_70 -35 val_70 -35 val_70 -35 val_70 -35 val_70 -35 val_70 -35 val_70 -35 val_70 -35 val_70 -35 val_70 -35 val_70 -37 val_74 -37 val_74 -37 val_74 -37 val_74 -4 val_8 -41 val_82 -42 val_84 -42 val_84 -42 val_84 -42 val_84 -42 val_84 -42 val_84 -42 val_84 -42 val_84 -43 val_86 Index: src/contrib/hive/ql/src/test/results/clientpositive/input3.q.out =================================================================== --- src/contrib/hive/ql/src/test/results/clientpositive/input3.q.out (revision 711472) +++ src/contrib/hive/ql/src/test/results/clientpositive/input3.q.out (working copy) @@ -61,4 +61,4 @@ r1 int r2 float Detailed Table Information: -Table(tableName:test3c,dbName:default,owner:njain,createTime:1224285029,lastAccessTime:0,retention:0,sd:StorageDescriptor(cols:[FieldSchema(name:r1,type:int,comment:null), FieldSchema(name:r2,type:float,comment:null)],location:file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/test3b,inputFormat:org.apache.hadoop.mapred.TextInputFormat,outputFormat:org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat,compressed:false,numBuckets:-1,serdeInfo:SerDeInfo(name:null,serializationLib:org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe,parameters:{serialization.format=1}),bucketCols:[],sortCols:[],parameters:{}),partitionKeys:[],parameters:{last_modified_by=njain,last_modified_time=1224285029,SORTBUCKETCOLSPREFIX=TRUE}) +Table(tableName:test3c,dbName:default,owner:athusoo,createTime:1225763946,lastAccessTime:0,retention:0,sd:StorageDescriptor(cols:[FieldSchema(name:r1,type:int,comment:null), FieldSchema(name:r2,type:float,comment:null)],location:file:/data/users/athusoo/hadoop_local_ws1/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/test3b,inputFormat:org.apache.hadoop.mapred.TextInputFormat,outputFormat:org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat,compressed:false,numBuckets:-1,serdeInfo:SerDeInfo(name:null,serializationLib:org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe,parameters:{serialization.format=1}),bucketCols:[],sortCols:[],parameters:{}),partitionKeys:[],parameters:{last_modified_by=athusoo,last_modified_time=1225763947}) Index: src/contrib/hive/ql/src/test/results/clientpositive/join4.q.out =================================================================== --- src/contrib/hive/ql/src/test/results/clientpositive/join4.q.out (revision 711472) +++ src/contrib/hive/ql/src/test/results/clientpositive/join4.q.out (working copy) @@ -13,7 +13,7 @@ Filter Operator predicate: expr: ((key > 15) and (key < 25)) - type: Boolean + type: boolean Select Operator expressions: expr: key @@ -35,7 +35,7 @@ Filter Operator predicate: expr: ((key > 10) and (key < 20)) - type: Boolean + type: boolean Select Operator expressions: expr: key Index: src/contrib/hive/ql/src/test/results/clientpositive/input5.q.out =================================================================== --- src/contrib/hive/ql/src/test/results/clientpositive/input5.q.out (revision 711472) +++ src/contrib/hive/ql/src/test/results/clientpositive/input5.q.out (working copy) @@ -1,5 +1,5 @@ ABSTRACT SYNTAX TREE: - (TOK_QUERY (TOK_FROM (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF src_thrift)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TRANSFORM (TOK_COLLIST (TOK_COLREF src_thrift lint) (TOK_COLREF src_thrift lintstring)) (TOK_ALIASLIST tkey tvalue) '/bin/cat'))) (TOK_CLUSTERBY tkey))) tmap)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB dest1)) (TOK_SELECT (TOK_SELEXPR (TOK_COLREF tmap tkey)) (TOK_SELEXPR (TOK_COLREF tmap tvalue))))) + (TOK_QUERY (TOK_FROM (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF src_thrift)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TRANSFORM (TOK_EXPLIST (TOK_COLREF src_thrift lint) (TOK_COLREF src_thrift lintstring)) (TOK_ALIASLIST tkey tvalue) '/bin/cat'))) (TOK_CLUSTERBY tkey))) tmap)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB dest1)) (TOK_SELECT (TOK_SELEXPR (TOK_COLREF tmap tkey)) (TOK_SELEXPR (TOK_COLREF tmap tvalue))))) STAGE DEPENDENCIES: Stage-1 is a root stage Index: src/contrib/hive/ql/src/test/results/clientpositive/join6.q.out =================================================================== --- src/contrib/hive/ql/src/test/results/clientpositive/join6.q.out (revision 711472) +++ src/contrib/hive/ql/src/test/results/clientpositive/join6.q.out (working copy) @@ -13,7 +13,7 @@ Filter Operator predicate: expr: ((key > 15) and (key < 25)) - type: Boolean + type: boolean Select Operator expressions: expr: key @@ -35,7 +35,7 @@ Filter Operator predicate: expr: ((key > 10) and (key < 20)) - type: Boolean + type: boolean Select Operator expressions: expr: key Index: src/contrib/hive/ql/src/test/results/clientpositive/input_dynamicserde.q.out =================================================================== --- src/contrib/hive/ql/src/test/results/clientpositive/input_dynamicserde.q.out (revision 711472) +++ src/contrib/hive/ql/src/test/results/clientpositive/input_dynamicserde.q.out (working copy) @@ -50,3 +50,13 @@ [7,14,21] ["70","700","7000"] {"key_7":"value_7"} 1539139264 record_7 [8,16,24] ["80","800","8000"] {"key_8":"value_8"} -1103622763 record_8 [9,18,27] ["90","900","9000"] {"key_9":"value_9"} -1883609167 record_9 +0 0 NULL -1220068486 record_0 +1 10 NULL -1147582750 record_1 +2 20 NULL -2091002570 record_2 +3 30 NULL -1587372273 record_3 +4 40 NULL -240543265 record_4 +5 50 NULL 1914724537 record_5 +6 60 NULL -1281615210 record_6 +7 70 NULL 1539139264 record_7 +8 80 NULL -1103622763 record_8 +9 90 NULL -1883609167 record_9 Index: src/contrib/hive/ql/src/test/results/clientpositive/join8.q.out =================================================================== --- src/contrib/hive/ql/src/test/results/clientpositive/join8.q.out (revision 711472) +++ src/contrib/hive/ql/src/test/results/clientpositive/join8.q.out (working copy) @@ -13,7 +13,7 @@ Filter Operator predicate: expr: ((key > 15) and (key < 25)) - type: Boolean + type: boolean Select Operator expressions: expr: key @@ -35,7 +35,7 @@ Filter Operator predicate: expr: ((key > 10) and (key < 20)) - type: Boolean + type: boolean Select Operator expressions: expr: key @@ -73,7 +73,7 @@ Filter Operator predicate: expr: (2 is null and 0 is not null) - type: Boolean + type: boolean Select Operator expressions: expr: 0 Index: src/contrib/hive/ql/src/test/results/clientpositive/input9.q.out =================================================================== --- src/contrib/hive/ql/src/test/results/clientpositive/input9.q.out (revision 711472) +++ src/contrib/hive/ql/src/test/results/clientpositive/input9.q.out (working copy) @@ -13,7 +13,7 @@ Filter Operator predicate: expr: (null = null) - type: Boolean + type: boolean Select Operator expressions: expr: null Index: src/contrib/hive/ql/src/test/results/clientpositive/udf1.q.out =================================================================== --- src/contrib/hive/ql/src/test/results/clientpositive/udf1.q.out (revision 711472) +++ src/contrib/hive/ql/src/test/results/clientpositive/udf1.q.out (working copy) @@ -13,35 +13,35 @@ Filter Operator predicate: expr: (key = 86) - type: Boolean + type: boolean Select Operator expressions: expr: ('a' like '%a%') - type: Boolean + type: boolean expr: ('b' like '%a%') - type: Boolean + type: boolean expr: ('ab' like '%a%') - type: Boolean + type: boolean expr: ('ab' like '%a_') - type: Boolean + type: boolean expr: ('%_' like '\%\_') - type: Boolean + type: boolean expr: ('ab' like '\%\_') - type: Boolean + type: boolean expr: ('ab' like '_a%') - type: Boolean + type: boolean expr: ('ab' like 'a') - type: Boolean + type: boolean expr: ('' regexp '.*') - type: Boolean + type: boolean expr: ('a' regexp '[ab]') - type: Boolean + type: boolean expr: ('' regexp '[ab]') - type: Boolean + type: boolean expr: ('hadoop' regexp '[a-z]*') - type: Boolean + type: boolean expr: ('hadoop' regexp 'o*') - type: Boolean + type: boolean expr: regexp_replace('abc', 'b', 'c') type: string expr: regexp_replace('abc', 'z', 'a') Index: src/contrib/hive/ql/src/test/results/clientpositive/input11.q.out =================================================================== --- src/contrib/hive/ql/src/test/results/clientpositive/input11.q.out (revision 711472) +++ src/contrib/hive/ql/src/test/results/clientpositive/input11.q.out (working copy) @@ -13,7 +13,7 @@ Filter Operator predicate: expr: (key < 100) - type: Boolean + type: boolean Select Operator expressions: expr: key Index: src/contrib/hive/ql/src/test/results/clientpositive/udf3.q.out =================================================================== --- src/contrib/hive/ql/src/test/results/clientpositive/udf3.q.out (revision 711472) +++ src/contrib/hive/ql/src/test/results/clientpositive/udf3.q.out (working copy) @@ -20,35 +20,36 @@ Reduce Operator Tree: Group By Operator - expr: max(VALUE.0) - expr: avg(VALUE.0) + expr: max(UDFToDouble(VALUE.0)) + expr: avg(UDFToDouble(VALUE.0)) expr: count(VALUE.0) - expr: sum(VALUE.0) - expr: min(VALUE.0) + expr: sum(UDFToDouble(VALUE.0)) + expr: min(UDFToDouble(VALUE.0)) mode: partial1 File Output Operator table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.mapred.SequenceFileOutputFormat + name: binary_table Stage: Stage-2 Map Reduce Alias -> Map Operator Tree: - /tmp/hive-njain/159338588/98912587.10001 + /tmp/hive-athusoo/1568776091/15337412.10001 Reduce Output Operator # partition fields: 0 tag: -1 value expressions: expr: 0 - type: string + type: double expr: 1 type: string expr: 2 - type: string + type: bigint expr: 3 - type: string + type: double expr: 4 - type: string + type: double Reduce Operator Tree: Group By Operator @@ -57,19 +58,19 @@ expr: count(VALUE.2) expr: sum(VALUE.3) expr: min(VALUE.4) - mode: partial2 + mode: unknown Select Operator expressions: expr: 2 - type: string + type: bigint expr: 3 - type: string + type: double expr: 1 type: string expr: 4 - type: string + type: double expr: 0 - type: string + type: double File Output Operator table: input format: org.apache.hadoop.mapred.TextInputFormat Index: src/contrib/hive/ql/src/test/results/clientpositive/noalias_subq1.q.out =================================================================== --- src/contrib/hive/ql/src/test/results/clientpositive/noalias_subq1.q.out (revision 0) +++ src/contrib/hive/ql/src/test/results/clientpositive/noalias_subq1.q.out (revision 0) @@ -0,0 +1,120 @@ +ABSTRACT SYNTAX TREE: + (TOK_QUERY (TOK_FROM (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF src)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_COLREF value) c1) (TOK_SELEXPR (TOK_COLREF key) c2)))) x)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_COLREF c1))) (TOK_WHERE (< (TOK_COLREF c2) 100)))) + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Alias -> Map Operator Tree: + x:src + Select Operator + expressions: + expr: value + type: string + expr: key + type: string + Filter Operator + predicate: + expr: (1 < 100) + type: boolean + Select Operator + expressions: + expr: 0 + type: string + File Output Operator + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + + Stage: Stage-0 + Fetch Operator + limit: -1 + + +val_86 +val_27 +val_98 +val_66 +val_37 +val_15 +val_82 +val_17 +val_0 +val_57 +val_20 +val_92 +val_47 +val_72 +val_4 +val_35 +val_54 +val_51 +val_65 +val_83 +val_12 +val_67 +val_84 +val_58 +val_8 +val_24 +val_42 +val_0 +val_96 +val_26 +val_51 +val_43 +val_95 +val_98 +val_85 +val_77 +val_0 +val_87 +val_15 +val_72 +val_90 +val_19 +val_10 +val_5 +val_58 +val_35 +val_95 +val_11 +val_34 +val_42 +val_78 +val_76 +val_41 +val_30 +val_64 +val_76 +val_74 +val_69 +val_33 +val_70 +val_5 +val_2 +val_35 +val_80 +val_44 +val_53 +val_90 +val_12 +val_5 +val_70 +val_24 +val_70 +val_83 +val_26 +val_67 +val_18 +val_9 +val_18 +val_97 +val_84 +val_28 +val_37 +val_90 +val_97 Index: src/contrib/hive/ql/src/test/results/clientpositive/input13.q.out =================================================================== --- src/contrib/hive/ql/src/test/results/clientpositive/input13.q.out (revision 711472) +++ src/contrib/hive/ql/src/test/results/clientpositive/input13.q.out (working copy) @@ -13,7 +13,7 @@ Filter Operator predicate: expr: (key < 100) - type: Boolean + type: boolean Select Operator expressions: expr: key @@ -29,7 +29,7 @@ Filter Operator predicate: expr: ((key >= 100) and (key < 200)) - type: Boolean + type: boolean Select Operator expressions: expr: key @@ -45,7 +45,7 @@ Filter Operator predicate: expr: ((key >= 200) and (key < 300)) - type: Boolean + type: boolean Select Operator expressions: expr: key @@ -59,7 +59,7 @@ Filter Operator predicate: expr: (key >= 300) - type: Boolean + type: boolean Select Operator expressions: expr: value @@ -287,109 +287,109 @@ 194 val_194 126 val_126 169 val_169 -238 NULL 2008-04-08 12 -255 NULL 2008-04-08 12 -278 NULL 2008-04-08 12 -265 NULL 2008-04-08 12 -273 NULL 2008-04-08 12 -224 NULL 2008-04-08 12 -213 NULL 2008-04-08 12 -281 NULL 2008-04-08 12 -277 NULL 2008-04-08 12 -209 NULL 2008-04-08 12 -252 NULL 2008-04-08 12 -292 NULL 2008-04-08 12 -219 NULL 2008-04-08 12 -287 NULL 2008-04-08 12 -237 NULL 2008-04-08 12 -207 NULL 2008-04-08 12 -208 NULL 2008-04-08 12 -247 NULL 2008-04-08 12 -266 NULL 2008-04-08 12 -203 NULL 2008-04-08 12 -205 NULL 2008-04-08 12 -221 NULL 2008-04-08 12 -280 NULL 2008-04-08 12 -277 NULL 2008-04-08 12 -208 NULL 2008-04-08 12 -286 NULL 2008-04-08 12 -239 NULL 2008-04-08 12 -213 NULL 2008-04-08 12 -216 NULL 2008-04-08 12 -278 NULL 2008-04-08 12 -289 NULL 2008-04-08 12 -221 NULL 2008-04-08 12 -275 NULL 2008-04-08 12 -241 NULL 2008-04-08 12 -284 NULL 2008-04-08 12 -230 NULL 2008-04-08 12 -260 NULL 2008-04-08 12 -272 NULL 2008-04-08 12 -217 NULL 2008-04-08 12 -230 NULL 2008-04-08 12 -208 NULL 2008-04-08 12 -298 NULL 2008-04-08 12 -230 NULL 2008-04-08 12 -205 NULL 2008-04-08 12 -288 NULL 2008-04-08 12 -282 NULL 2008-04-08 12 -282 NULL 2008-04-08 12 -238 NULL 2008-04-08 12 -277 NULL 2008-04-08 12 -273 NULL 2008-04-08 12 -224 NULL 2008-04-08 12 -242 NULL 2008-04-08 12 -272 NULL 2008-04-08 12 -242 NULL 2008-04-08 12 -226 NULL 2008-04-08 12 -229 NULL 2008-04-08 12 -233 NULL 2008-04-08 12 -223 NULL 2008-04-08 12 -218 NULL 2008-04-08 12 -228 NULL 2008-04-08 12 -209 NULL 2008-04-08 12 -230 NULL 2008-04-08 12 -296 NULL 2008-04-08 12 -216 NULL 2008-04-08 12 -274 NULL 2008-04-08 12 -219 NULL 2008-04-08 12 -239 NULL 2008-04-08 12 -223 NULL 2008-04-08 12 -256 NULL 2008-04-08 12 -263 NULL 2008-04-08 12 -288 NULL 2008-04-08 12 -244 NULL 2008-04-08 12 -202 NULL 2008-04-08 12 -229 NULL 2008-04-08 12 -280 NULL 2008-04-08 12 -283 NULL 2008-04-08 12 -235 NULL 2008-04-08 12 -257 NULL 2008-04-08 12 -258 NULL 2008-04-08 12 -203 NULL 2008-04-08 12 -262 NULL 2008-04-08 12 -201 NULL 2008-04-08 12 -217 NULL 2008-04-08 12 -298 NULL 2008-04-08 12 -291 NULL 2008-04-08 12 -255 NULL 2008-04-08 12 -200 NULL 2008-04-08 12 -237 NULL 2008-04-08 12 -248 NULL 2008-04-08 12 -277 NULL 2008-04-08 12 -230 NULL 2008-04-08 12 -207 NULL 2008-04-08 12 -249 NULL 2008-04-08 12 -265 NULL 2008-04-08 12 -214 NULL 2008-04-08 12 -233 NULL 2008-04-08 12 -256 NULL 2008-04-08 12 -298 NULL 2008-04-08 12 -285 NULL 2008-04-08 12 -273 NULL 2008-04-08 12 -281 NULL 2008-04-08 12 -222 NULL 2008-04-08 12 -200 NULL 2008-04-08 12 +238 2008-04-08 12 +255 2008-04-08 12 +278 2008-04-08 12 +265 2008-04-08 12 +273 2008-04-08 12 +224 2008-04-08 12 +213 2008-04-08 12 +281 2008-04-08 12 +277 2008-04-08 12 +209 2008-04-08 12 +252 2008-04-08 12 +292 2008-04-08 12 +219 2008-04-08 12 +287 2008-04-08 12 +237 2008-04-08 12 +207 2008-04-08 12 +208 2008-04-08 12 +247 2008-04-08 12 +266 2008-04-08 12 +203 2008-04-08 12 +205 2008-04-08 12 +221 2008-04-08 12 +280 2008-04-08 12 +277 2008-04-08 12 +208 2008-04-08 12 +286 2008-04-08 12 +239 2008-04-08 12 +213 2008-04-08 12 +216 2008-04-08 12 +278 2008-04-08 12 +289 2008-04-08 12 +221 2008-04-08 12 +275 2008-04-08 12 +241 2008-04-08 12 +284 2008-04-08 12 +230 2008-04-08 12 +260 2008-04-08 12 +272 2008-04-08 12 +217 2008-04-08 12 +230 2008-04-08 12 +208 2008-04-08 12 +298 2008-04-08 12 +230 2008-04-08 12 +205 2008-04-08 12 +288 2008-04-08 12 +282 2008-04-08 12 +282 2008-04-08 12 +238 2008-04-08 12 +277 2008-04-08 12 +273 2008-04-08 12 +224 2008-04-08 12 +242 2008-04-08 12 +272 2008-04-08 12 +242 2008-04-08 12 +226 2008-04-08 12 +229 2008-04-08 12 +233 2008-04-08 12 +223 2008-04-08 12 +218 2008-04-08 12 +228 2008-04-08 12 +209 2008-04-08 12 +230 2008-04-08 12 +296 2008-04-08 12 +216 2008-04-08 12 +274 2008-04-08 12 +219 2008-04-08 12 +239 2008-04-08 12 +223 2008-04-08 12 +256 2008-04-08 12 +263 2008-04-08 12 +288 2008-04-08 12 +244 2008-04-08 12 +202 2008-04-08 12 +229 2008-04-08 12 +280 2008-04-08 12 +283 2008-04-08 12 +235 2008-04-08 12 +257 2008-04-08 12 +258 2008-04-08 12 +203 2008-04-08 12 +262 2008-04-08 12 +201 2008-04-08 12 +217 2008-04-08 12 +298 2008-04-08 12 +291 2008-04-08 12 +255 2008-04-08 12 +200 2008-04-08 12 +237 2008-04-08 12 +248 2008-04-08 12 +277 2008-04-08 12 +230 2008-04-08 12 +207 2008-04-08 12 +249 2008-04-08 12 +265 2008-04-08 12 +214 2008-04-08 12 +233 2008-04-08 12 +256 2008-04-08 12 +298 2008-04-08 12 +285 2008-04-08 12 +273 2008-04-08 12 +281 2008-04-08 12 +222 2008-04-08 12 +200 2008-04-08 12 val_311 val_409 val_484 Index: src/contrib/hive/ql/src/test/results/clientpositive/input15.q.out =================================================================== --- src/contrib/hive/ql/src/test/results/clientpositive/input15.q.out (revision 711472) +++ src/contrib/hive/ql/src/test/results/clientpositive/input15.q.out (working copy) @@ -1,5 +1,5 @@ ABSTRACT SYNTAX TREE: - (TOK_CREATETABLE TEST15 (TOK_TABCOLLIST (TOK_TABCOL key TOK_INT) (TOK_TABCOL value TOK_STRING)) (TOK_TABLEROWFORMAT (TOK_TABLEROWFORMATFIELD '\t'))) + (TOK_CREATETABLE TEST15 (TOK_TABCOLLIST (TOK_TABCOL key TOK_INT) (TOK_TABCOL value TOK_STRING)) (TOK_TABLEROWFORMAT (TOK_TABLEROWFORMATFIELD '\t')) TOK_TBLTEXTFILE) STAGE DEPENDENCIES: Stage-0 is a root stage Index: src/contrib/hive/ql/src/test/results/clientpositive/input_part0.q.out =================================================================== --- src/contrib/hive/ql/src/test/results/clientpositive/input_part0.q.out (revision 0) +++ src/contrib/hive/ql/src/test/results/clientpositive/input_part0.q.out (revision 0) @@ -0,0 +1,1012 @@ +ABSTRACT SYNTAX TREE: + (TOK_QUERY (TOK_FROM (TOK_TABREF SRCPART x)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_ALLCOLREF x))) (TOK_WHERE (= (TOK_COLREF x ds) '2008-04-08')))) + +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + + +238 val_238 2008-04-08 11 +86 val_86 2008-04-08 11 +311 val_311 2008-04-08 11 +27 val_27 2008-04-08 11 +165 val_165 2008-04-08 11 +409 val_409 2008-04-08 11 +255 val_255 2008-04-08 11 +278 val_278 2008-04-08 11 +98 val_98 2008-04-08 11 +484 val_484 2008-04-08 11 +265 val_265 2008-04-08 11 +193 val_193 2008-04-08 11 +401 val_401 2008-04-08 11 +150 val_150 2008-04-08 11 +273 val_273 2008-04-08 11 +224 val_224 2008-04-08 11 +369 val_369 2008-04-08 11 +66 val_66 2008-04-08 11 +128 val_128 2008-04-08 11 +213 val_213 2008-04-08 11 +146 val_146 2008-04-08 11 +406 val_406 2008-04-08 11 +429 val_429 2008-04-08 11 +374 val_374 2008-04-08 11 +152 val_152 2008-04-08 11 +469 val_469 2008-04-08 11 +145 val_145 2008-04-08 11 +495 val_495 2008-04-08 11 +37 val_37 2008-04-08 11 +327 val_327 2008-04-08 11 +281 val_281 2008-04-08 11 +277 val_277 2008-04-08 11 +209 val_209 2008-04-08 11 +15 val_15 2008-04-08 11 +82 val_82 2008-04-08 11 +403 val_403 2008-04-08 11 +166 val_166 2008-04-08 11 +417 val_417 2008-04-08 11 +430 val_430 2008-04-08 11 +252 val_252 2008-04-08 11 +292 val_292 2008-04-08 11 +219 val_219 2008-04-08 11 +287 val_287 2008-04-08 11 +153 val_153 2008-04-08 11 +193 val_193 2008-04-08 11 +338 val_338 2008-04-08 11 +446 val_446 2008-04-08 11 +459 val_459 2008-04-08 11 +394 val_394 2008-04-08 11 +237 val_237 2008-04-08 11 +482 val_482 2008-04-08 11 +174 val_174 2008-04-08 11 +413 val_413 2008-04-08 11 +494 val_494 2008-04-08 11 +207 val_207 2008-04-08 11 +199 val_199 2008-04-08 11 +466 val_466 2008-04-08 11 +208 val_208 2008-04-08 11 +174 val_174 2008-04-08 11 +399 val_399 2008-04-08 11 +396 val_396 2008-04-08 11 +247 val_247 2008-04-08 11 +417 val_417 2008-04-08 11 +489 val_489 2008-04-08 11 +162 val_162 2008-04-08 11 +377 val_377 2008-04-08 11 +397 val_397 2008-04-08 11 +309 val_309 2008-04-08 11 +365 val_365 2008-04-08 11 +266 val_266 2008-04-08 11 +439 val_439 2008-04-08 11 +342 val_342 2008-04-08 11 +367 val_367 2008-04-08 11 +325 val_325 2008-04-08 11 +167 val_167 2008-04-08 11 +195 val_195 2008-04-08 11 +475 val_475 2008-04-08 11 +17 val_17 2008-04-08 11 +113 val_113 2008-04-08 11 +155 val_155 2008-04-08 11 +203 val_203 2008-04-08 11 +339 val_339 2008-04-08 11 +0 val_0 2008-04-08 11 +455 val_455 2008-04-08 11 +128 val_128 2008-04-08 11 +311 val_311 2008-04-08 11 +316 val_316 2008-04-08 11 +57 val_57 2008-04-08 11 +302 val_302 2008-04-08 11 +205 val_205 2008-04-08 11 +149 val_149 2008-04-08 11 +438 val_438 2008-04-08 11 +345 val_345 2008-04-08 11 +129 val_129 2008-04-08 11 +170 val_170 2008-04-08 11 +20 val_20 2008-04-08 11 +489 val_489 2008-04-08 11 +157 val_157 2008-04-08 11 +378 val_378 2008-04-08 11 +221 val_221 2008-04-08 11 +92 val_92 2008-04-08 11 +111 val_111 2008-04-08 11 +47 val_47 2008-04-08 11 +72 val_72 2008-04-08 11 +4 val_4 2008-04-08 11 +280 val_280 2008-04-08 11 +35 val_35 2008-04-08 11 +427 val_427 2008-04-08 11 +277 val_277 2008-04-08 11 +208 val_208 2008-04-08 11 +356 val_356 2008-04-08 11 +399 val_399 2008-04-08 11 +169 val_169 2008-04-08 11 +382 val_382 2008-04-08 11 +498 val_498 2008-04-08 11 +125 val_125 2008-04-08 11 +386 val_386 2008-04-08 11 +437 val_437 2008-04-08 11 +469 val_469 2008-04-08 11 +192 val_192 2008-04-08 11 +286 val_286 2008-04-08 11 +187 val_187 2008-04-08 11 +176 val_176 2008-04-08 11 +54 val_54 2008-04-08 11 +459 val_459 2008-04-08 11 +51 val_51 2008-04-08 11 +138 val_138 2008-04-08 11 +103 val_103 2008-04-08 11 +239 val_239 2008-04-08 11 +213 val_213 2008-04-08 11 +216 val_216 2008-04-08 11 +430 val_430 2008-04-08 11 +278 val_278 2008-04-08 11 +176 val_176 2008-04-08 11 +289 val_289 2008-04-08 11 +221 val_221 2008-04-08 11 +65 val_65 2008-04-08 11 +318 val_318 2008-04-08 11 +332 val_332 2008-04-08 11 +311 val_311 2008-04-08 11 +275 val_275 2008-04-08 11 +137 val_137 2008-04-08 11 +241 val_241 2008-04-08 11 +83 val_83 2008-04-08 11 +333 val_333 2008-04-08 11 +180 val_180 2008-04-08 11 +284 val_284 2008-04-08 11 +12 val_12 2008-04-08 11 +230 val_230 2008-04-08 11 +181 val_181 2008-04-08 11 +67 val_67 2008-04-08 11 +260 val_260 2008-04-08 11 +404 val_404 2008-04-08 11 +384 val_384 2008-04-08 11 +489 val_489 2008-04-08 11 +353 val_353 2008-04-08 11 +373 val_373 2008-04-08 11 +272 val_272 2008-04-08 11 +138 val_138 2008-04-08 11 +217 val_217 2008-04-08 11 +84 val_84 2008-04-08 11 +348 val_348 2008-04-08 11 +466 val_466 2008-04-08 11 +58 val_58 2008-04-08 11 +8 val_8 2008-04-08 11 +411 val_411 2008-04-08 11 +230 val_230 2008-04-08 11 +208 val_208 2008-04-08 11 +348 val_348 2008-04-08 11 +24 val_24 2008-04-08 11 +463 val_463 2008-04-08 11 +431 val_431 2008-04-08 11 +179 val_179 2008-04-08 11 +172 val_172 2008-04-08 11 +42 val_42 2008-04-08 11 +129 val_129 2008-04-08 11 +158 val_158 2008-04-08 11 +119 val_119 2008-04-08 11 +496 val_496 2008-04-08 11 +0 val_0 2008-04-08 11 +322 val_322 2008-04-08 11 +197 val_197 2008-04-08 11 +468 val_468 2008-04-08 11 +393 val_393 2008-04-08 11 +454 val_454 2008-04-08 11 +100 val_100 2008-04-08 11 +298 val_298 2008-04-08 11 +199 val_199 2008-04-08 11 +191 val_191 2008-04-08 11 +418 val_418 2008-04-08 11 +96 val_96 2008-04-08 11 +26 val_26 2008-04-08 11 +165 val_165 2008-04-08 11 +327 val_327 2008-04-08 11 +230 val_230 2008-04-08 11 +205 val_205 2008-04-08 11 +120 val_120 2008-04-08 11 +131 val_131 2008-04-08 11 +51 val_51 2008-04-08 11 +404 val_404 2008-04-08 11 +43 val_43 2008-04-08 11 +436 val_436 2008-04-08 11 +156 val_156 2008-04-08 11 +469 val_469 2008-04-08 11 +468 val_468 2008-04-08 11 +308 val_308 2008-04-08 11 +95 val_95 2008-04-08 11 +196 val_196 2008-04-08 11 +288 val_288 2008-04-08 11 +481 val_481 2008-04-08 11 +457 val_457 2008-04-08 11 +98 val_98 2008-04-08 11 +282 val_282 2008-04-08 11 +197 val_197 2008-04-08 11 +187 val_187 2008-04-08 11 +318 val_318 2008-04-08 11 +318 val_318 2008-04-08 11 +409 val_409 2008-04-08 11 +470 val_470 2008-04-08 11 +137 val_137 2008-04-08 11 +369 val_369 2008-04-08 11 +316 val_316 2008-04-08 11 +169 val_169 2008-04-08 11 +413 val_413 2008-04-08 11 +85 val_85 2008-04-08 11 +77 val_77 2008-04-08 11 +0 val_0 2008-04-08 11 +490 val_490 2008-04-08 11 +87 val_87 2008-04-08 11 +364 val_364 2008-04-08 11 +179 val_179 2008-04-08 11 +118 val_118 2008-04-08 11 +134 val_134 2008-04-08 11 +395 val_395 2008-04-08 11 +282 val_282 2008-04-08 11 +138 val_138 2008-04-08 11 +238 val_238 2008-04-08 11 +419 val_419 2008-04-08 11 +15 val_15 2008-04-08 11 +118 val_118 2008-04-08 11 +72 val_72 2008-04-08 11 +90 val_90 2008-04-08 11 +307 val_307 2008-04-08 11 +19 val_19 2008-04-08 11 +435 val_435 2008-04-08 11 +10 val_10 2008-04-08 11 +277 val_277 2008-04-08 11 +273 val_273 2008-04-08 11 +306 val_306 2008-04-08 11 +224 val_224 2008-04-08 11 +309 val_309 2008-04-08 11 +389 val_389 2008-04-08 11 +327 val_327 2008-04-08 11 +242 val_242 2008-04-08 11 +369 val_369 2008-04-08 11 +392 val_392 2008-04-08 11 +272 val_272 2008-04-08 11 +331 val_331 2008-04-08 11 +401 val_401 2008-04-08 11 +242 val_242 2008-04-08 11 +452 val_452 2008-04-08 11 +177 val_177 2008-04-08 11 +226 val_226 2008-04-08 11 +5 val_5 2008-04-08 11 +497 val_497 2008-04-08 11 +402 val_402 2008-04-08 11 +396 val_396 2008-04-08 11 +317 val_317 2008-04-08 11 +395 val_395 2008-04-08 11 +58 val_58 2008-04-08 11 +35 val_35 2008-04-08 11 +336 val_336 2008-04-08 11 +95 val_95 2008-04-08 11 +11 val_11 2008-04-08 11 +168 val_168 2008-04-08 11 +34 val_34 2008-04-08 11 +229 val_229 2008-04-08 11 +233 val_233 2008-04-08 11 +143 val_143 2008-04-08 11 +472 val_472 2008-04-08 11 +322 val_322 2008-04-08 11 +498 val_498 2008-04-08 11 +160 val_160 2008-04-08 11 +195 val_195 2008-04-08 11 +42 val_42 2008-04-08 11 +321 val_321 2008-04-08 11 +430 val_430 2008-04-08 11 +119 val_119 2008-04-08 11 +489 val_489 2008-04-08 11 +458 val_458 2008-04-08 11 +78 val_78 2008-04-08 11 +76 val_76 2008-04-08 11 +41 val_41 2008-04-08 11 +223 val_223 2008-04-08 11 +492 val_492 2008-04-08 11 +149 val_149 2008-04-08 11 +449 val_449 2008-04-08 11 +218 val_218 2008-04-08 11 +228 val_228 2008-04-08 11 +138 val_138 2008-04-08 11 +453 val_453 2008-04-08 11 +30 val_30 2008-04-08 11 +209 val_209 2008-04-08 11 +64 val_64 2008-04-08 11 +468 val_468 2008-04-08 11 +76 val_76 2008-04-08 11 +74 val_74 2008-04-08 11 +342 val_342 2008-04-08 11 +69 val_69 2008-04-08 11 +230 val_230 2008-04-08 11 +33 val_33 2008-04-08 11 +368 val_368 2008-04-08 11 +103 val_103 2008-04-08 11 +296 val_296 2008-04-08 11 +113 val_113 2008-04-08 11 +216 val_216 2008-04-08 11 +367 val_367 2008-04-08 11 +344 val_344 2008-04-08 11 +167 val_167 2008-04-08 11 +274 val_274 2008-04-08 11 +219 val_219 2008-04-08 11 +239 val_239 2008-04-08 11 +485 val_485 2008-04-08 11 +116 val_116 2008-04-08 11 +223 val_223 2008-04-08 11 +256 val_256 2008-04-08 11 +263 val_263 2008-04-08 11 +70 val_70 2008-04-08 11 +487 val_487 2008-04-08 11 +480 val_480 2008-04-08 11 +401 val_401 2008-04-08 11 +288 val_288 2008-04-08 11 +191 val_191 2008-04-08 11 +5 val_5 2008-04-08 11 +244 val_244 2008-04-08 11 +438 val_438 2008-04-08 11 +128 val_128 2008-04-08 11 +467 val_467 2008-04-08 11 +432 val_432 2008-04-08 11 +202 val_202 2008-04-08 11 +316 val_316 2008-04-08 11 +229 val_229 2008-04-08 11 +469 val_469 2008-04-08 11 +463 val_463 2008-04-08 11 +280 val_280 2008-04-08 11 +2 val_2 2008-04-08 11 +35 val_35 2008-04-08 11 +283 val_283 2008-04-08 11 +331 val_331 2008-04-08 11 +235 val_235 2008-04-08 11 +80 val_80 2008-04-08 11 +44 val_44 2008-04-08 11 +193 val_193 2008-04-08 11 +321 val_321 2008-04-08 11 +335 val_335 2008-04-08 11 +104 val_104 2008-04-08 11 +466 val_466 2008-04-08 11 +366 val_366 2008-04-08 11 +175 val_175 2008-04-08 11 +403 val_403 2008-04-08 11 +483 val_483 2008-04-08 11 +53 val_53 2008-04-08 11 +105 val_105 2008-04-08 11 +257 val_257 2008-04-08 11 +406 val_406 2008-04-08 11 +409 val_409 2008-04-08 11 +190 val_190 2008-04-08 11 +406 val_406 2008-04-08 11 +401 val_401 2008-04-08 11 +114 val_114 2008-04-08 11 +258 val_258 2008-04-08 11 +90 val_90 2008-04-08 11 +203 val_203 2008-04-08 11 +262 val_262 2008-04-08 11 +348 val_348 2008-04-08 11 +424 val_424 2008-04-08 11 +12 val_12 2008-04-08 11 +396 val_396 2008-04-08 11 +201 val_201 2008-04-08 11 +217 val_217 2008-04-08 11 +164 val_164 2008-04-08 11 +431 val_431 2008-04-08 11 +454 val_454 2008-04-08 11 +478 val_478 2008-04-08 11 +298 val_298 2008-04-08 11 +125 val_125 2008-04-08 11 +431 val_431 2008-04-08 11 +164 val_164 2008-04-08 11 +424 val_424 2008-04-08 11 +187 val_187 2008-04-08 11 +382 val_382 2008-04-08 11 +5 val_5 2008-04-08 11 +70 val_70 2008-04-08 11 +397 val_397 2008-04-08 11 +480 val_480 2008-04-08 11 +291 val_291 2008-04-08 11 +24 val_24 2008-04-08 11 +351 val_351 2008-04-08 11 +255 val_255 2008-04-08 11 +104 val_104 2008-04-08 11 +70 val_70 2008-04-08 11 +163 val_163 2008-04-08 11 +438 val_438 2008-04-08 11 +119 val_119 2008-04-08 11 +414 val_414 2008-04-08 11 +200 val_200 2008-04-08 11 +491 val_491 2008-04-08 11 +237 val_237 2008-04-08 11 +439 val_439 2008-04-08 11 +360 val_360 2008-04-08 11 +248 val_248 2008-04-08 11 +479 val_479 2008-04-08 11 +305 val_305 2008-04-08 11 +417 val_417 2008-04-08 11 +199 val_199 2008-04-08 11 +444 val_444 2008-04-08 11 +120 val_120 2008-04-08 11 +429 val_429 2008-04-08 11 +169 val_169 2008-04-08 11 +443 val_443 2008-04-08 11 +323 val_323 2008-04-08 11 +325 val_325 2008-04-08 11 +277 val_277 2008-04-08 11 +230 val_230 2008-04-08 11 +478 val_478 2008-04-08 11 +178 val_178 2008-04-08 11 +468 val_468 2008-04-08 11 +310 val_310 2008-04-08 11 +317 val_317 2008-04-08 11 +333 val_333 2008-04-08 11 +493 val_493 2008-04-08 11 +460 val_460 2008-04-08 11 +207 val_207 2008-04-08 11 +249 val_249 2008-04-08 11 +265 val_265 2008-04-08 11 +480 val_480 2008-04-08 11 +83 val_83 2008-04-08 11 +136 val_136 2008-04-08 11 +353 val_353 2008-04-08 11 +172 val_172 2008-04-08 11 +214 val_214 2008-04-08 11 +462 val_462 2008-04-08 11 +233 val_233 2008-04-08 11 +406 val_406 2008-04-08 11 +133 val_133 2008-04-08 11 +175 val_175 2008-04-08 11 +189 val_189 2008-04-08 11 +454 val_454 2008-04-08 11 +375 val_375 2008-04-08 11 +401 val_401 2008-04-08 11 +421 val_421 2008-04-08 11 +407 val_407 2008-04-08 11 +384 val_384 2008-04-08 11 +256 val_256 2008-04-08 11 +26 val_26 2008-04-08 11 +134 val_134 2008-04-08 11 +67 val_67 2008-04-08 11 +384 val_384 2008-04-08 11 +379 val_379 2008-04-08 11 +18 val_18 2008-04-08 11 +462 val_462 2008-04-08 11 +492 val_492 2008-04-08 11 +100 val_100 2008-04-08 11 +298 val_298 2008-04-08 11 +9 val_9 2008-04-08 11 +341 val_341 2008-04-08 11 +498 val_498 2008-04-08 11 +146 val_146 2008-04-08 11 +458 val_458 2008-04-08 11 +362 val_362 2008-04-08 11 +186 val_186 2008-04-08 11 +285 val_285 2008-04-08 11 +348 val_348 2008-04-08 11 +167 val_167 2008-04-08 11 +18 val_18 2008-04-08 11 +273 val_273 2008-04-08 11 +183 val_183 2008-04-08 11 +281 val_281 2008-04-08 11 +344 val_344 2008-04-08 11 +97 val_97 2008-04-08 11 +469 val_469 2008-04-08 11 +315 val_315 2008-04-08 11 +84 val_84 2008-04-08 11 +28 val_28 2008-04-08 11 +37 val_37 2008-04-08 11 +448 val_448 2008-04-08 11 +152 val_152 2008-04-08 11 +348 val_348 2008-04-08 11 +307 val_307 2008-04-08 11 +194 val_194 2008-04-08 11 +414 val_414 2008-04-08 11 +477 val_477 2008-04-08 11 +222 val_222 2008-04-08 11 +126 val_126 2008-04-08 11 +90 val_90 2008-04-08 11 +169 val_169 2008-04-08 11 +403 val_403 2008-04-08 11 +400 val_400 2008-04-08 11 +200 val_200 2008-04-08 11 +97 val_97 2008-04-08 11 +238 val_238 2008-04-08 12 +86 val_86 2008-04-08 12 +311 val_311 2008-04-08 12 +27 val_27 2008-04-08 12 +165 val_165 2008-04-08 12 +409 val_409 2008-04-08 12 +255 val_255 2008-04-08 12 +278 val_278 2008-04-08 12 +98 val_98 2008-04-08 12 +484 val_484 2008-04-08 12 +265 val_265 2008-04-08 12 +193 val_193 2008-04-08 12 +401 val_401 2008-04-08 12 +150 val_150 2008-04-08 12 +273 val_273 2008-04-08 12 +224 val_224 2008-04-08 12 +369 val_369 2008-04-08 12 +66 val_66 2008-04-08 12 +128 val_128 2008-04-08 12 +213 val_213 2008-04-08 12 +146 val_146 2008-04-08 12 +406 val_406 2008-04-08 12 +429 val_429 2008-04-08 12 +374 val_374 2008-04-08 12 +152 val_152 2008-04-08 12 +469 val_469 2008-04-08 12 +145 val_145 2008-04-08 12 +495 val_495 2008-04-08 12 +37 val_37 2008-04-08 12 +327 val_327 2008-04-08 12 +281 val_281 2008-04-08 12 +277 val_277 2008-04-08 12 +209 val_209 2008-04-08 12 +15 val_15 2008-04-08 12 +82 val_82 2008-04-08 12 +403 val_403 2008-04-08 12 +166 val_166 2008-04-08 12 +417 val_417 2008-04-08 12 +430 val_430 2008-04-08 12 +252 val_252 2008-04-08 12 +292 val_292 2008-04-08 12 +219 val_219 2008-04-08 12 +287 val_287 2008-04-08 12 +153 val_153 2008-04-08 12 +193 val_193 2008-04-08 12 +338 val_338 2008-04-08 12 +446 val_446 2008-04-08 12 +459 val_459 2008-04-08 12 +394 val_394 2008-04-08 12 +237 val_237 2008-04-08 12 +482 val_482 2008-04-08 12 +174 val_174 2008-04-08 12 +413 val_413 2008-04-08 12 +494 val_494 2008-04-08 12 +207 val_207 2008-04-08 12 +199 val_199 2008-04-08 12 +466 val_466 2008-04-08 12 +208 val_208 2008-04-08 12 +174 val_174 2008-04-08 12 +399 val_399 2008-04-08 12 +396 val_396 2008-04-08 12 +247 val_247 2008-04-08 12 +417 val_417 2008-04-08 12 +489 val_489 2008-04-08 12 +162 val_162 2008-04-08 12 +377 val_377 2008-04-08 12 +397 val_397 2008-04-08 12 +309 val_309 2008-04-08 12 +365 val_365 2008-04-08 12 +266 val_266 2008-04-08 12 +439 val_439 2008-04-08 12 +342 val_342 2008-04-08 12 +367 val_367 2008-04-08 12 +325 val_325 2008-04-08 12 +167 val_167 2008-04-08 12 +195 val_195 2008-04-08 12 +475 val_475 2008-04-08 12 +17 val_17 2008-04-08 12 +113 val_113 2008-04-08 12 +155 val_155 2008-04-08 12 +203 val_203 2008-04-08 12 +339 val_339 2008-04-08 12 +0 val_0 2008-04-08 12 +455 val_455 2008-04-08 12 +128 val_128 2008-04-08 12 +311 val_311 2008-04-08 12 +316 val_316 2008-04-08 12 +57 val_57 2008-04-08 12 +302 val_302 2008-04-08 12 +205 val_205 2008-04-08 12 +149 val_149 2008-04-08 12 +438 val_438 2008-04-08 12 +345 val_345 2008-04-08 12 +129 val_129 2008-04-08 12 +170 val_170 2008-04-08 12 +20 val_20 2008-04-08 12 +489 val_489 2008-04-08 12 +157 val_157 2008-04-08 12 +378 val_378 2008-04-08 12 +221 val_221 2008-04-08 12 +92 val_92 2008-04-08 12 +111 val_111 2008-04-08 12 +47 val_47 2008-04-08 12 +72 val_72 2008-04-08 12 +4 val_4 2008-04-08 12 +280 val_280 2008-04-08 12 +35 val_35 2008-04-08 12 +427 val_427 2008-04-08 12 +277 val_277 2008-04-08 12 +208 val_208 2008-04-08 12 +356 val_356 2008-04-08 12 +399 val_399 2008-04-08 12 +169 val_169 2008-04-08 12 +382 val_382 2008-04-08 12 +498 val_498 2008-04-08 12 +125 val_125 2008-04-08 12 +386 val_386 2008-04-08 12 +437 val_437 2008-04-08 12 +469 val_469 2008-04-08 12 +192 val_192 2008-04-08 12 +286 val_286 2008-04-08 12 +187 val_187 2008-04-08 12 +176 val_176 2008-04-08 12 +54 val_54 2008-04-08 12 +459 val_459 2008-04-08 12 +51 val_51 2008-04-08 12 +138 val_138 2008-04-08 12 +103 val_103 2008-04-08 12 +239 val_239 2008-04-08 12 +213 val_213 2008-04-08 12 +216 val_216 2008-04-08 12 +430 val_430 2008-04-08 12 +278 val_278 2008-04-08 12 +176 val_176 2008-04-08 12 +289 val_289 2008-04-08 12 +221 val_221 2008-04-08 12 +65 val_65 2008-04-08 12 +318 val_318 2008-04-08 12 +332 val_332 2008-04-08 12 +311 val_311 2008-04-08 12 +275 val_275 2008-04-08 12 +137 val_137 2008-04-08 12 +241 val_241 2008-04-08 12 +83 val_83 2008-04-08 12 +333 val_333 2008-04-08 12 +180 val_180 2008-04-08 12 +284 val_284 2008-04-08 12 +12 val_12 2008-04-08 12 +230 val_230 2008-04-08 12 +181 val_181 2008-04-08 12 +67 val_67 2008-04-08 12 +260 val_260 2008-04-08 12 +404 val_404 2008-04-08 12 +384 val_384 2008-04-08 12 +489 val_489 2008-04-08 12 +353 val_353 2008-04-08 12 +373 val_373 2008-04-08 12 +272 val_272 2008-04-08 12 +138 val_138 2008-04-08 12 +217 val_217 2008-04-08 12 +84 val_84 2008-04-08 12 +348 val_348 2008-04-08 12 +466 val_466 2008-04-08 12 +58 val_58 2008-04-08 12 +8 val_8 2008-04-08 12 +411 val_411 2008-04-08 12 +230 val_230 2008-04-08 12 +208 val_208 2008-04-08 12 +348 val_348 2008-04-08 12 +24 val_24 2008-04-08 12 +463 val_463 2008-04-08 12 +431 val_431 2008-04-08 12 +179 val_179 2008-04-08 12 +172 val_172 2008-04-08 12 +42 val_42 2008-04-08 12 +129 val_129 2008-04-08 12 +158 val_158 2008-04-08 12 +119 val_119 2008-04-08 12 +496 val_496 2008-04-08 12 +0 val_0 2008-04-08 12 +322 val_322 2008-04-08 12 +197 val_197 2008-04-08 12 +468 val_468 2008-04-08 12 +393 val_393 2008-04-08 12 +454 val_454 2008-04-08 12 +100 val_100 2008-04-08 12 +298 val_298 2008-04-08 12 +199 val_199 2008-04-08 12 +191 val_191 2008-04-08 12 +418 val_418 2008-04-08 12 +96 val_96 2008-04-08 12 +26 val_26 2008-04-08 12 +165 val_165 2008-04-08 12 +327 val_327 2008-04-08 12 +230 val_230 2008-04-08 12 +205 val_205 2008-04-08 12 +120 val_120 2008-04-08 12 +131 val_131 2008-04-08 12 +51 val_51 2008-04-08 12 +404 val_404 2008-04-08 12 +43 val_43 2008-04-08 12 +436 val_436 2008-04-08 12 +156 val_156 2008-04-08 12 +469 val_469 2008-04-08 12 +468 val_468 2008-04-08 12 +308 val_308 2008-04-08 12 +95 val_95 2008-04-08 12 +196 val_196 2008-04-08 12 +288 val_288 2008-04-08 12 +481 val_481 2008-04-08 12 +457 val_457 2008-04-08 12 +98 val_98 2008-04-08 12 +282 val_282 2008-04-08 12 +197 val_197 2008-04-08 12 +187 val_187 2008-04-08 12 +318 val_318 2008-04-08 12 +318 val_318 2008-04-08 12 +409 val_409 2008-04-08 12 +470 val_470 2008-04-08 12 +137 val_137 2008-04-08 12 +369 val_369 2008-04-08 12 +316 val_316 2008-04-08 12 +169 val_169 2008-04-08 12 +413 val_413 2008-04-08 12 +85 val_85 2008-04-08 12 +77 val_77 2008-04-08 12 +0 val_0 2008-04-08 12 +490 val_490 2008-04-08 12 +87 val_87 2008-04-08 12 +364 val_364 2008-04-08 12 +179 val_179 2008-04-08 12 +118 val_118 2008-04-08 12 +134 val_134 2008-04-08 12 +395 val_395 2008-04-08 12 +282 val_282 2008-04-08 12 +138 val_138 2008-04-08 12 +238 val_238 2008-04-08 12 +419 val_419 2008-04-08 12 +15 val_15 2008-04-08 12 +118 val_118 2008-04-08 12 +72 val_72 2008-04-08 12 +90 val_90 2008-04-08 12 +307 val_307 2008-04-08 12 +19 val_19 2008-04-08 12 +435 val_435 2008-04-08 12 +10 val_10 2008-04-08 12 +277 val_277 2008-04-08 12 +273 val_273 2008-04-08 12 +306 val_306 2008-04-08 12 +224 val_224 2008-04-08 12 +309 val_309 2008-04-08 12 +389 val_389 2008-04-08 12 +327 val_327 2008-04-08 12 +242 val_242 2008-04-08 12 +369 val_369 2008-04-08 12 +392 val_392 2008-04-08 12 +272 val_272 2008-04-08 12 +331 val_331 2008-04-08 12 +401 val_401 2008-04-08 12 +242 val_242 2008-04-08 12 +452 val_452 2008-04-08 12 +177 val_177 2008-04-08 12 +226 val_226 2008-04-08 12 +5 val_5 2008-04-08 12 +497 val_497 2008-04-08 12 +402 val_402 2008-04-08 12 +396 val_396 2008-04-08 12 +317 val_317 2008-04-08 12 +395 val_395 2008-04-08 12 +58 val_58 2008-04-08 12 +35 val_35 2008-04-08 12 +336 val_336 2008-04-08 12 +95 val_95 2008-04-08 12 +11 val_11 2008-04-08 12 +168 val_168 2008-04-08 12 +34 val_34 2008-04-08 12 +229 val_229 2008-04-08 12 +233 val_233 2008-04-08 12 +143 val_143 2008-04-08 12 +472 val_472 2008-04-08 12 +322 val_322 2008-04-08 12 +498 val_498 2008-04-08 12 +160 val_160 2008-04-08 12 +195 val_195 2008-04-08 12 +42 val_42 2008-04-08 12 +321 val_321 2008-04-08 12 +430 val_430 2008-04-08 12 +119 val_119 2008-04-08 12 +489 val_489 2008-04-08 12 +458 val_458 2008-04-08 12 +78 val_78 2008-04-08 12 +76 val_76 2008-04-08 12 +41 val_41 2008-04-08 12 +223 val_223 2008-04-08 12 +492 val_492 2008-04-08 12 +149 val_149 2008-04-08 12 +449 val_449 2008-04-08 12 +218 val_218 2008-04-08 12 +228 val_228 2008-04-08 12 +138 val_138 2008-04-08 12 +453 val_453 2008-04-08 12 +30 val_30 2008-04-08 12 +209 val_209 2008-04-08 12 +64 val_64 2008-04-08 12 +468 val_468 2008-04-08 12 +76 val_76 2008-04-08 12 +74 val_74 2008-04-08 12 +342 val_342 2008-04-08 12 +69 val_69 2008-04-08 12 +230 val_230 2008-04-08 12 +33 val_33 2008-04-08 12 +368 val_368 2008-04-08 12 +103 val_103 2008-04-08 12 +296 val_296 2008-04-08 12 +113 val_113 2008-04-08 12 +216 val_216 2008-04-08 12 +367 val_367 2008-04-08 12 +344 val_344 2008-04-08 12 +167 val_167 2008-04-08 12 +274 val_274 2008-04-08 12 +219 val_219 2008-04-08 12 +239 val_239 2008-04-08 12 +485 val_485 2008-04-08 12 +116 val_116 2008-04-08 12 +223 val_223 2008-04-08 12 +256 val_256 2008-04-08 12 +263 val_263 2008-04-08 12 +70 val_70 2008-04-08 12 +487 val_487 2008-04-08 12 +480 val_480 2008-04-08 12 +401 val_401 2008-04-08 12 +288 val_288 2008-04-08 12 +191 val_191 2008-04-08 12 +5 val_5 2008-04-08 12 +244 val_244 2008-04-08 12 +438 val_438 2008-04-08 12 +128 val_128 2008-04-08 12 +467 val_467 2008-04-08 12 +432 val_432 2008-04-08 12 +202 val_202 2008-04-08 12 +316 val_316 2008-04-08 12 +229 val_229 2008-04-08 12 +469 val_469 2008-04-08 12 +463 val_463 2008-04-08 12 +280 val_280 2008-04-08 12 +2 val_2 2008-04-08 12 +35 val_35 2008-04-08 12 +283 val_283 2008-04-08 12 +331 val_331 2008-04-08 12 +235 val_235 2008-04-08 12 +80 val_80 2008-04-08 12 +44 val_44 2008-04-08 12 +193 val_193 2008-04-08 12 +321 val_321 2008-04-08 12 +335 val_335 2008-04-08 12 +104 val_104 2008-04-08 12 +466 val_466 2008-04-08 12 +366 val_366 2008-04-08 12 +175 val_175 2008-04-08 12 +403 val_403 2008-04-08 12 +483 val_483 2008-04-08 12 +53 val_53 2008-04-08 12 +105 val_105 2008-04-08 12 +257 val_257 2008-04-08 12 +406 val_406 2008-04-08 12 +409 val_409 2008-04-08 12 +190 val_190 2008-04-08 12 +406 val_406 2008-04-08 12 +401 val_401 2008-04-08 12 +114 val_114 2008-04-08 12 +258 val_258 2008-04-08 12 +90 val_90 2008-04-08 12 +203 val_203 2008-04-08 12 +262 val_262 2008-04-08 12 +348 val_348 2008-04-08 12 +424 val_424 2008-04-08 12 +12 val_12 2008-04-08 12 +396 val_396 2008-04-08 12 +201 val_201 2008-04-08 12 +217 val_217 2008-04-08 12 +164 val_164 2008-04-08 12 +431 val_431 2008-04-08 12 +454 val_454 2008-04-08 12 +478 val_478 2008-04-08 12 +298 val_298 2008-04-08 12 +125 val_125 2008-04-08 12 +431 val_431 2008-04-08 12 +164 val_164 2008-04-08 12 +424 val_424 2008-04-08 12 +187 val_187 2008-04-08 12 +382 val_382 2008-04-08 12 +5 val_5 2008-04-08 12 +70 val_70 2008-04-08 12 +397 val_397 2008-04-08 12 +480 val_480 2008-04-08 12 +291 val_291 2008-04-08 12 +24 val_24 2008-04-08 12 +351 val_351 2008-04-08 12 +255 val_255 2008-04-08 12 +104 val_104 2008-04-08 12 +70 val_70 2008-04-08 12 +163 val_163 2008-04-08 12 +438 val_438 2008-04-08 12 +119 val_119 2008-04-08 12 +414 val_414 2008-04-08 12 +200 val_200 2008-04-08 12 +491 val_491 2008-04-08 12 +237 val_237 2008-04-08 12 +439 val_439 2008-04-08 12 +360 val_360 2008-04-08 12 +248 val_248 2008-04-08 12 +479 val_479 2008-04-08 12 +305 val_305 2008-04-08 12 +417 val_417 2008-04-08 12 +199 val_199 2008-04-08 12 +444 val_444 2008-04-08 12 +120 val_120 2008-04-08 12 +429 val_429 2008-04-08 12 +169 val_169 2008-04-08 12 +443 val_443 2008-04-08 12 +323 val_323 2008-04-08 12 +325 val_325 2008-04-08 12 +277 val_277 2008-04-08 12 +230 val_230 2008-04-08 12 +478 val_478 2008-04-08 12 +178 val_178 2008-04-08 12 +468 val_468 2008-04-08 12 +310 val_310 2008-04-08 12 +317 val_317 2008-04-08 12 +333 val_333 2008-04-08 12 +493 val_493 2008-04-08 12 +460 val_460 2008-04-08 12 +207 val_207 2008-04-08 12 +249 val_249 2008-04-08 12 +265 val_265 2008-04-08 12 +480 val_480 2008-04-08 12 +83 val_83 2008-04-08 12 +136 val_136 2008-04-08 12 +353 val_353 2008-04-08 12 +172 val_172 2008-04-08 12 +214 val_214 2008-04-08 12 +462 val_462 2008-04-08 12 +233 val_233 2008-04-08 12 +406 val_406 2008-04-08 12 +133 val_133 2008-04-08 12 +175 val_175 2008-04-08 12 +189 val_189 2008-04-08 12 +454 val_454 2008-04-08 12 +375 val_375 2008-04-08 12 +401 val_401 2008-04-08 12 +421 val_421 2008-04-08 12 +407 val_407 2008-04-08 12 +384 val_384 2008-04-08 12 +256 val_256 2008-04-08 12 +26 val_26 2008-04-08 12 +134 val_134 2008-04-08 12 +67 val_67 2008-04-08 12 +384 val_384 2008-04-08 12 +379 val_379 2008-04-08 12 +18 val_18 2008-04-08 12 +462 val_462 2008-04-08 12 +492 val_492 2008-04-08 12 +100 val_100 2008-04-08 12 +298 val_298 2008-04-08 12 +9 val_9 2008-04-08 12 +341 val_341 2008-04-08 12 +498 val_498 2008-04-08 12 +146 val_146 2008-04-08 12 +458 val_458 2008-04-08 12 +362 val_362 2008-04-08 12 +186 val_186 2008-04-08 12 +285 val_285 2008-04-08 12 +348 val_348 2008-04-08 12 +167 val_167 2008-04-08 12 +18 val_18 2008-04-08 12 +273 val_273 2008-04-08 12 +183 val_183 2008-04-08 12 +281 val_281 2008-04-08 12 +344 val_344 2008-04-08 12 +97 val_97 2008-04-08 12 +469 val_469 2008-04-08 12 +315 val_315 2008-04-08 12 +84 val_84 2008-04-08 12 +28 val_28 2008-04-08 12 +37 val_37 2008-04-08 12 +448 val_448 2008-04-08 12 +152 val_152 2008-04-08 12 +348 val_348 2008-04-08 12 +307 val_307 2008-04-08 12 +194 val_194 2008-04-08 12 +414 val_414 2008-04-08 12 +477 val_477 2008-04-08 12 +222 val_222 2008-04-08 12 +126 val_126 2008-04-08 12 +90 val_90 2008-04-08 12 +169 val_169 2008-04-08 12 +403 val_403 2008-04-08 12 +400 val_400 2008-04-08 12 +200 val_200 2008-04-08 12 +97 val_97 2008-04-08 12 Index: src/contrib/hive/ql/src/test/results/clientpositive/input_part2.q.out =================================================================== --- src/contrib/hive/ql/src/test/results/clientpositive/input_part2.q.out (revision 711472) +++ src/contrib/hive/ql/src/test/results/clientpositive/input_part2.q.out (working copy) @@ -13,7 +13,7 @@ Filter Operator predicate: expr: (((key < 100) and (ds = '2008-04-08')) and (hr = '12')) - type: Boolean + type: boolean Select Operator expressions: expr: key @@ -25,7 +25,7 @@ expr: ds type: string File Output Operator - directory: /tmp/hive-njain/925269835/11972654.10000.insclause-0 + directory: /tmp/hive-athusoo/319920467/273834772.10000.insclause-0 table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat @@ -34,18 +34,17 @@ serialization.ddl struct dest1 { i32 key, string value, string hr, string ds} serialization.format 1 columns key,value,hr,ds - SORTBUCKETCOLSPREFIX TRUE bucket_count -1 serialization.lib org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat - location file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/dest1 + location file:/data/users/athusoo/hadoop_local_ws1/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/dest1 serde: org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe name: dest1 Filter Operator predicate: expr: (((key < 100) and (ds = '2008-04-09')) and (hr = '12')) - type: Boolean + type: boolean Select Operator expressions: expr: key @@ -57,7 +56,7 @@ expr: ds type: string File Output Operator - directory: /tmp/hive-njain/925269835/11972654.10001.insclause-1 + directory: /tmp/hive-athusoo/319920467/273834772.10001.insclause-1 table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat @@ -66,20 +65,19 @@ serialization.ddl struct dest2 { i32 key, string value, string hr, string ds} serialization.format 1 columns key,value,hr,ds - SORTBUCKETCOLSPREFIX TRUE bucket_count -1 serialization.lib org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat - location file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/dest2 + location file:/data/users/athusoo/hadoop_local_ws1/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/dest2 serde: org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe name: dest2 Needs Tagging: Path -> Alias: - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/srcpart/ds=2008-04-08/hr=12 - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/srcpart/ds=2008-04-09/hr=12 + file:/data/users/athusoo/hadoop_local_ws1/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/srcpart/ds=2008-04-08/hr=12 + file:/data/users/athusoo/hadoop_local_ws1/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/srcpart/ds=2008-04-09/hr=12 Path -> Partition: - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/srcpart/ds=2008-04-08/hr=12 + file:/data/users/athusoo/hadoop_local_ws1/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/srcpart/ds=2008-04-08/hr=12 Partition partition values: ds 2008-04-08 @@ -97,10 +95,10 @@ serialization.lib org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat - location file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/srcpart + location file:/data/users/athusoo/hadoop_local_ws1/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/srcpart serde: org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe name: srcpart - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/srcpart/ds=2008-04-09/hr=12 + file:/data/users/athusoo/hadoop_local_ws1/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/srcpart/ds=2008-04-09/hr=12 Partition partition values: ds 2008-04-09 @@ -118,7 +116,7 @@ serialization.lib org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat - location file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/srcpart + location file:/data/users/athusoo/hadoop_local_ws1/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/srcpart serde: org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe name: srcpart @@ -126,7 +124,7 @@ Move Operator tables: replace: - source: /tmp/hive-njain/925269835/11972654.10000.insclause-0 + source: /tmp/hive-athusoo/319920467/273834772.10000.insclause-0 table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat @@ -135,16 +133,15 @@ serialization.ddl struct dest1 { i32 key, string value, string hr, string ds} serialization.format 1 columns key,value,hr,ds - SORTBUCKETCOLSPREFIX TRUE bucket_count -1 serialization.lib org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat - location file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/dest1 + location file:/data/users/athusoo/hadoop_local_ws1/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/dest1 serde: org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe name: dest1 replace: - source: /tmp/hive-njain/925269835/11972654.10001.insclause-1 + source: /tmp/hive-athusoo/319920467/273834772.10001.insclause-1 table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat @@ -153,12 +150,11 @@ serialization.ddl struct dest2 { i32 key, string value, string hr, string ds} serialization.format 1 columns key,value,hr,ds - SORTBUCKETCOLSPREFIX TRUE bucket_count -1 serialization.lib org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat - location file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/dest2 + location file:/data/users/athusoo/hadoop_local_ws1/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/dest2 serde: org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe name: dest2 Index: src/contrib/hive/ql/src/test/results/clientpositive/input17.q.out =================================================================== --- src/contrib/hive/ql/src/test/results/clientpositive/input17.q.out (revision 0) +++ src/contrib/hive/ql/src/test/results/clientpositive/input17.q.out (revision 0) @@ -0,0 +1,70 @@ +ABSTRACT SYNTAX TREE: + (TOK_QUERY (TOK_FROM (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF src_thrift)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TRANSFORM (TOK_EXPLIST (+ (TOK_COLREF src_thrift aint) ([ (TOK_COLREF src_thrift lint) 0)) ([ (TOK_COLREF src_thrift lintstring) 0)) (TOK_ALIASLIST tkey tvalue) '/bin/cat'))) (TOK_CLUSTERBY tkey))) tmap)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB dest1)) (TOK_SELECT (TOK_SELEXPR (TOK_COLREF tmap tkey)) (TOK_SELEXPR (TOK_COLREF tmap tvalue))))) + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Alias -> Map Operator Tree: + tmap:src_thrift + Select Operator + expressions: + expr: (aint + lint[0]) + type: int + expr: lintstring[0] + type: struct{myint:int,mystring:string} + Transform Operator + command: /bin/cat + output info: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + Reduce Output Operator + key expressions: + expr: tkey + type: string + # partition fields: 1 + tag: -1 + value expressions: + expr: tkey + type: string + expr: tvalue + type: string + Reduce Operator Tree: + Extract + Select Operator + expressions: + expr: 0 + type: string + expr: 1 + type: string + File Output Operator + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe + name: dest1 + + Stage: Stage-0 + Move Operator + tables: + replace: + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe + name: dest1 + + +-1103622755 {"myint":64,"mystring":"512"} +-1147582749 {"myint":1,"mystring":"1"} +-1220068486 {"myint":0,"mystring":"0"} +-1281615204 {"myint":36,"mystring":"216"} +-1587372270 {"myint":9,"mystring":"27"} +-1883609158 {"myint":81,"mystring":"729"} +-2091002568 {"myint":4,"mystring":"8"} +-240543261 {"myint":16,"mystring":"64"} +1539139271 {"myint":49,"mystring":"343"} +1914724542 {"myint":25,"mystring":"125"} Index: src/contrib/hive/ql/src/test/results/clientpositive/groupby1.q.out =================================================================== --- src/contrib/hive/ql/src/test/results/clientpositive/groupby1.q.out (revision 711472) +++ src/contrib/hive/ql/src/test/results/clientpositive/groupby1.q.out (working copy) @@ -23,20 +23,21 @@ Reduce Operator Tree: Group By Operator - expr: sum(VALUE.0) + expr: sum(UDFToDouble(VALUE.0)) keys: expr: KEY.0 type: string mode: partial1 File Output Operator table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.mapred.SequenceFileOutputFormat + name: binary_table Stage: Stage-2 Map Reduce Alias -> Map Operator Tree: - /tmp/hive-njain/748679827/1407352694.10001 + /tmp/hive-athusoo/1043210550/859291804.10001 Reduce Output Operator key expressions: expr: 0 @@ -45,7 +46,7 @@ tag: -1 value expressions: expr: 1 - type: string + type: double Reduce Operator Tree: Group By Operator @@ -53,13 +54,13 @@ keys: expr: KEY.0 type: string - mode: partial2 + mode: unknown Select Operator expressions: expr: 0 type: string expr: 1 - type: string + type: double File Output Operator table: input format: org.apache.hadoop.mapred.TextInputFormat Index: src/contrib/hive/ql/src/test/results/clientpositive/input_part4.q.out =================================================================== --- src/contrib/hive/ql/src/test/results/clientpositive/input_part4.q.out (revision 0) +++ src/contrib/hive/ql/src/test/results/clientpositive/input_part4.q.out (revision 0) @@ -0,0 +1,12 @@ +ABSTRACT SYNTAX TREE: + (TOK_QUERY (TOK_FROM (TOK_TABREF SRCPART x)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_ALLCOLREF x))) (TOK_WHERE (and (= (TOK_COLREF x ds) '2008-04-08') (= (TOK_COLREF x hr) 15))))) + +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + + Index: src/contrib/hive/ql/src/test/results/clientpositive/groupby3.q.out =================================================================== --- src/contrib/hive/ql/src/test/results/clientpositive/groupby3.q.out (revision 711472) +++ src/contrib/hive/ql/src/test/results/clientpositive/groupby3.q.out (working copy) @@ -20,21 +20,22 @@ Reduce Operator Tree: Group By Operator - expr: avg(DISTINCT KEY.0) - expr: sum(KEY.0) - expr: avg(KEY.0) - expr: min(KEY.0) - expr: max(KEY.0) + expr: avg(DISTINCT UDFToDouble(KEY.0)) + expr: sum(UDFToDouble(KEY.0)) + expr: avg(UDFToDouble(KEY.0)) + expr: min(UDFToDouble(KEY.0)) + expr: max(UDFToDouble(KEY.0)) mode: partial1 File Output Operator table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.mapred.SequenceFileOutputFormat + name: binary_table Stage: Stage-2 Map Reduce Alias -> Map Operator Tree: - /tmp/hive-njain/629715569/118113569.10001 + /tmp/hive-athusoo/76213411/833455542.10001 Reduce Output Operator # partition fields: 0 tag: -1 @@ -42,13 +43,13 @@ expr: 0 type: string expr: 1 - type: string + type: double expr: 2 type: string expr: 3 - type: string + type: double expr: 4 - type: string + type: double Reduce Operator Tree: Group By Operator @@ -57,19 +58,19 @@ expr: avg(VALUE.2) expr: min(VALUE.3) expr: max(VALUE.4) - mode: partial2 + mode: unknown Select Operator expressions: expr: 1 - type: string + type: double expr: 2 type: string expr: 0 type: string expr: 4 - type: string + type: double expr: 3 - type: string + type: double File Output Operator table: input format: org.apache.hadoop.mapred.TextInputFormat Index: src/contrib/hive/ql/src/test/results/clientpositive/subq.q.out =================================================================== --- src/contrib/hive/ql/src/test/results/clientpositive/subq.q.out (revision 711472) +++ src/contrib/hive/ql/src/test/results/clientpositive/subq.q.out (working copy) @@ -13,7 +13,7 @@ Filter Operator predicate: expr: (key < 100) - type: Boolean + type: boolean Select Operator expressions: expr: key Index: src/contrib/hive/ql/src/test/results/clientpositive/groupby5.q.out =================================================================== --- src/contrib/hive/ql/src/test/results/clientpositive/groupby5.q.out (revision 711472) +++ src/contrib/hive/ql/src/test/results/clientpositive/groupby5.q.out (working copy) @@ -23,20 +23,21 @@ Reduce Operator Tree: Group By Operator - expr: sum(VALUE.0) + expr: sum(UDFToDouble(VALUE.0)) keys: expr: KEY.0 type: string mode: partial1 File Output Operator table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.mapred.SequenceFileOutputFormat + name: binary_table Stage: Stage-2 Map Reduce Alias -> Map Operator Tree: - /tmp/hive-njain/888102295/1013886705.10001 + /tmp/hive-athusoo/213675/984446175.10001 Reduce Output Operator key expressions: expr: 0 @@ -45,7 +46,7 @@ tag: -1 value expressions: expr: 1 - type: string + type: double Reduce Operator Tree: Group By Operator @@ -53,13 +54,13 @@ keys: expr: KEY.0 type: string - mode: partial2 + mode: unknown Select Operator expressions: expr: 0 type: string expr: 1 - type: string + type: double File Output Operator table: input format: org.apache.hadoop.mapred.TextInputFormat Index: src/contrib/hive/ql/src/test/results/clientpositive/alter1.q.out =================================================================== --- src/contrib/hive/ql/src/test/results/clientpositive/alter1.q.out (revision 0) +++ src/contrib/hive/ql/src/test/results/clientpositive/alter1.q.out (revision 0) @@ -0,0 +1,28 @@ +a int +b int +Detailed Table Information: +Table(tableName:alter1,dbName:default,owner:athusoo,createTime:1225764063,lastAccessTime:0,retention:0,sd:StorageDescriptor(cols:[FieldSchema(name:a,type:int,comment:null), FieldSchema(name:b,type:int,comment:null)],location:file:/data/users/athusoo/hadoop_local_ws1/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/alter1,inputFormat:org.apache.hadoop.mapred.TextInputFormat,outputFormat:org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat,compressed:false,numBuckets:-1,serdeInfo:SerDeInfo(name:null,serializationLib:org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe,parameters:{serialization.format=1}),bucketCols:[],sortCols:[],parameters:{}),partitionKeys:[],parameters:{}) +a int +b int +Detailed Table Information: +Table(tableName:alter1,dbName:default,owner:athusoo,createTime:1225764063,lastAccessTime:0,retention:0,sd:StorageDescriptor(cols:[FieldSchema(name:a,type:int,comment:null), FieldSchema(name:b,type:int,comment:null)],location:file:/data/users/athusoo/hadoop_local_ws1/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/alter1,inputFormat:org.apache.hadoop.mapred.TextInputFormat,outputFormat:org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat,compressed:false,numBuckets:-1,serdeInfo:SerDeInfo(name:null,serializationLib:org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe,parameters:{serialization.format=1}),bucketCols:[],sortCols:[],parameters:{}),partitionKeys:[],parameters:{last_modified_by=athusoo,c=3,last_modified_time=1225764063,a=1}) +a int +b int +Detailed Table Information: +Table(tableName:alter1,dbName:default,owner:athusoo,createTime:1225764063,lastAccessTime:0,retention:0,sd:StorageDescriptor(cols:[FieldSchema(name:a,type:int,comment:null), FieldSchema(name:b,type:int,comment:null)],location:file:/data/users/athusoo/hadoop_local_ws1/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/alter1,inputFormat:org.apache.hadoop.mapred.TextInputFormat,outputFormat:org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat,compressed:false,numBuckets:-1,serdeInfo:SerDeInfo(name:null,serializationLib:org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe,parameters:{serialization.format=1}),bucketCols:[],sortCols:[],parameters:{}),partitionKeys:[],parameters:{d=3,last_modified_by=athusoo,c=4,last_modified_time=1225764063,a=1}) +a int +b int +Detailed Table Information: +Table(tableName:alter1,dbName:default,owner:athusoo,createTime:1225764063,lastAccessTime:0,retention:0,sd:StorageDescriptor(cols:[FieldSchema(name:a,type:int,comment:null), FieldSchema(name:b,type:int,comment:null)],location:file:/data/users/athusoo/hadoop_local_ws1/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/alter1,inputFormat:org.apache.hadoop.mapred.TextInputFormat,outputFormat:org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat,compressed:false,numBuckets:-1,serdeInfo:SerDeInfo(name:null,serializationLib:org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe,parameters:{s1=9,serialization.format=1}),bucketCols:[],sortCols:[],parameters:{}),partitionKeys:[],parameters:{d=3,last_modified_by=athusoo,c=4,last_modified_time=1225764063,a=1}) +a int +b int +Detailed Table Information: +Table(tableName:alter1,dbName:default,owner:athusoo,createTime:1225764063,lastAccessTime:0,retention:0,sd:StorageDescriptor(cols:[FieldSchema(name:a,type:int,comment:null), FieldSchema(name:b,type:int,comment:null)],location:file:/data/users/athusoo/hadoop_local_ws1/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/alter1,inputFormat:org.apache.hadoop.mapred.TextInputFormat,outputFormat:org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat,compressed:false,numBuckets:-1,serdeInfo:SerDeInfo(name:null,serializationLib:org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe,parameters:{s2=20,s1=10,serialization.format=1}),bucketCols:[],sortCols:[],parameters:{}),partitionKeys:[],parameters:{d=3,last_modified_by=athusoo,c=4,last_modified_time=1225764063,a=1}) +a string 'from deserializer' +b string 'from deserializer' +Detailed Table Information: +Table(tableName:alter1,dbName:default,owner:athusoo,createTime:1225764063,lastAccessTime:0,retention:0,sd:StorageDescriptor(cols:[FieldSchema(name:a,type:string,comment:from deserializer), FieldSchema(name:b,type:string,comment:from deserializer)],location:file:/data/users/athusoo/hadoop_local_ws1/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/alter1,inputFormat:org.apache.hadoop.mapred.TextInputFormat,outputFormat:org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat,compressed:false,numBuckets:-1,serdeInfo:SerDeInfo(name:null,serializationLib:org.apache.hadoop.hive.serde2.TestSerDe,parameters:{s2=20,s1=9,serialization.format=1}),bucketCols:[],sortCols:[],parameters:{}),partitionKeys:[],parameters:{d=3,last_modified_by=athusoo,c=4,last_modified_time=1225764063,a=1}) +a string 'from deserializer' +b string 'from deserializer' +Detailed Table Information: +Table(tableName:alter1,dbName:default,owner:athusoo,createTime:1225764063,lastAccessTime:0,retention:0,sd:StorageDescriptor(cols:[FieldSchema(name:a,type:string,comment:from deserializer), FieldSchema(name:b,type:string,comment:from deserializer)],location:file:/data/users/athusoo/hadoop_local_ws1/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/alter1,inputFormat:org.apache.hadoop.mapred.TextInputFormat,outputFormat:org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat,compressed:false,numBuckets:-1,serdeInfo:SerDeInfo(name:null,serializationLib:org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe,parameters:{s2=20,s1=9,serialization.format=1}),bucketCols:[],sortCols:[],parameters:{}),partitionKeys:[],parameters:{d=3,last_modified_by=athusoo,c=4,last_modified_time=1225764063,a=1}) Index: src/contrib/hive/ql/src/test/results/clientpositive/subq2.q.out =================================================================== --- src/contrib/hive/ql/src/test/results/clientpositive/subq2.q.out (revision 711472) +++ src/contrib/hive/ql/src/test/results/clientpositive/subq2.q.out (working copy) @@ -30,13 +30,14 @@ mode: partial1 File Output Operator table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.mapred.SequenceFileOutputFormat + name: binary_table Stage: Stage-2 Map Reduce Alias -> Map Operator Tree: - /tmp/hive-njain/57219117/75379653.10002 + /tmp/hive-athusoo/35862625/332262957.10002 Reduce Output Operator key expressions: expr: 0 @@ -45,7 +46,7 @@ tag: -1 value expressions: expr: 1 - type: string + type: bigint Reduce Operator Tree: Group By Operator @@ -53,23 +54,23 @@ keys: expr: KEY.0 type: string - mode: partial2 + mode: unknown Select Operator expressions: expr: 0 type: string expr: 1 - type: string + type: bigint Filter Operator predicate: expr: (0 >= 90) - type: Boolean + type: boolean Select Operator expressions: expr: 0 type: string expr: 1 - type: string + type: bigint File Output Operator table: input format: org.apache.hadoop.mapred.TextInputFormat Index: src/contrib/hive/ql/src/test/results/clientpositive/input11_limit.q.out =================================================================== --- src/contrib/hive/ql/src/test/results/clientpositive/input11_limit.q.out (revision 711472) +++ src/contrib/hive/ql/src/test/results/clientpositive/input11_limit.q.out (working copy) @@ -13,7 +13,7 @@ Filter Operator predicate: expr: (key < 100) - type: Boolean + type: boolean Select Operator expressions: expr: key Index: src/contrib/hive/ql/src/test/results/clientpositive/input14_limit.q.out =================================================================== --- src/contrib/hive/ql/src/test/results/clientpositive/input14_limit.q.out (revision 711472) +++ src/contrib/hive/ql/src/test/results/clientpositive/input14_limit.q.out (working copy) @@ -1,5 +1,5 @@ ABSTRACT SYNTAX TREE: - (TOK_QUERY (TOK_FROM (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF src)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TRANSFORM (TOK_COLLIST (TOK_COLREF src key) (TOK_COLREF src value)) (TOK_ALIASLIST tkey tvalue) '/bin/cat'))) (TOK_CLUSTERBY tkey) (TOK_LIMIT 20))) tmap)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB dest1)) (TOK_SELECT (TOK_SELEXPR (TOK_COLREF tmap tkey)) (TOK_SELEXPR (TOK_COLREF tmap tvalue))) (TOK_WHERE (< (TOK_COLREF tmap tkey) 100)))) + (TOK_QUERY (TOK_FROM (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF src)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TRANSFORM (TOK_EXPLIST (TOK_COLREF src key) (TOK_COLREF src value)) (TOK_ALIASLIST tkey tvalue) '/bin/cat'))) (TOK_CLUSTERBY tkey) (TOK_LIMIT 20))) tmap)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB dest1)) (TOK_SELECT (TOK_SELEXPR (TOK_COLREF tmap tkey)) (TOK_SELEXPR (TOK_COLREF tmap tvalue))) (TOK_WHERE (< (TOK_COLREF tmap tkey) 100)))) STAGE DEPENDENCIES: Stage-1 is a root stage @@ -38,13 +38,14 @@ Limit File Output Operator table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.mapred.SequenceFileOutputFormat + name: binary_table Stage: Stage-2 Map Reduce Alias -> Map Operator Tree: - /tmp/hive-njain/945772770/1814909502.10001 + /tmp/hive-athusoo/585012397/601049433.10001 Reduce Output Operator key expressions: expr: 0 @@ -63,7 +64,7 @@ Filter Operator predicate: expr: (0 < 100) - type: Boolean + type: boolean Select Operator expressions: expr: 0 Index: src/contrib/hive/ql/src/test/results/clientpositive/sample2.q.out =================================================================== --- src/contrib/hive/ql/src/test/results/clientpositive/sample2.q.out (revision 711472) +++ src/contrib/hive/ql/src/test/results/clientpositive/sample2.q.out (working copy) @@ -17,7 +17,7 @@ expr: value type: string File Output Operator - directory: /tmp/hive-njain/1178661057/1120939755.10000.insclause-0 + directory: /tmp/hive-athusoo/92409996/628171353.10000.insclause-0 table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat @@ -26,33 +26,33 @@ serialization.ddl struct dest1 { i32 key, string value} serialization.format 1 columns key,value - SORTBUCKETCOLSPREFIX TRUE bucket_count -1 serialization.lib org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat - location file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/dest1 + location file:/data/users/athusoo/hadoop_local_ws1/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/dest1 serde: org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe name: dest1 Needs Tagging: Path -> Alias: - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/srcbucket/kv1.txt + file:/data/users/athusoo/hadoop_local_ws1/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/srcbucket/kv1.txt Path -> Partition: - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/srcbucket/kv1.txt + file:/data/users/athusoo/hadoop_local_ws1/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/srcbucket/kv1.txt Partition input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat properties: name srcbucket + bucket_field_name key serialization.ddl struct srcbucket { string key, string value} - serialization.format 1 columns key,value + serialization.format 1 bucket_count 2 serialization.lib org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat - location file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/srcbucket + location file:/data/users/athusoo/hadoop_local_ws1/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/srcbucket serde: org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe name: srcbucket @@ -60,7 +60,7 @@ Move Operator tables: replace: - source: /tmp/hive-njain/1178661057/1120939755.10000.insclause-0 + source: /tmp/hive-athusoo/92409996/628171353.10000.insclause-0 table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat @@ -69,12 +69,11 @@ serialization.ddl struct dest1 { i32 key, string value} serialization.format 1 columns key,value - SORTBUCKETCOLSPREFIX TRUE bucket_count -1 serialization.lib org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat - location file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/dest1 + location file:/data/users/athusoo/hadoop_local_ws1/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/dest1 serde: org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe name: dest1 Index: src/contrib/hive/ql/src/test/results/clientpositive/inputddl1.q.out =================================================================== --- src/contrib/hive/ql/src/test/results/clientpositive/inputddl1.q.out (revision 711472) +++ src/contrib/hive/ql/src/test/results/clientpositive/inputddl1.q.out (working copy) @@ -1,5 +1,5 @@ ABSTRACT SYNTAX TREE: - (TOK_CREATETABLE INPUTDDL1 (TOK_TABCOLLIST (TOK_TABCOL key TOK_INT) (TOK_TABCOL value TOK_STRING))) + (TOK_CREATETABLE INPUTDDL1 (TOK_TABCOLLIST (TOK_TABCOL key TOK_INT) (TOK_TABCOL value TOK_STRING)) TOK_TBLTEXTFILE) STAGE DEPENDENCIES: Stage-0 is a root stage Index: src/contrib/hive/ql/src/test/results/clientpositive/sample4.q.out =================================================================== --- src/contrib/hive/ql/src/test/results/clientpositive/sample4.q.out (revision 711472) +++ src/contrib/hive/ql/src/test/results/clientpositive/sample4.q.out (working copy) @@ -1,5 +1,5 @@ ABSTRACT SYNTAX TREE: - (TOK_QUERY (TOK_FROM (TOK_TABREF srcbucket (TOK_TABLESAMPLE 1 2 key) s)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB dest1)) (TOK_SELECT (TOK_SELEXPR (TOK_ALLCOLREF s))))) + (TOK_QUERY (TOK_FROM (TOK_TABREF srcbucket (TOK_TABLESAMPLE 1 2 (TOK_COLREF key)) s)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB dest1)) (TOK_SELECT (TOK_SELEXPR (TOK_ALLCOLREF s))))) STAGE DEPENDENCIES: Stage-1 is a root stage @@ -10,53 +10,49 @@ Map Reduce Alias -> Map Operator Tree: s - Filter Operator - predicate: - expr: (((default_sample_hashfn(key) & 2147483647) % 2) = 0) - type: Boolean - Select Operator - expressions: - expr: key - type: string - expr: value - type: string - File Output Operator - directory: /tmp/hive-njain/594156852/317367185.10000.insclause-0 - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat - properties: - name dest1 - serialization.ddl struct dest1 { i32 key, string value} - serialization.format 1 - columns key,value - SORTBUCKETCOLSPREFIX TRUE - bucket_count -1 - serialization.lib org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe - file.inputformat org.apache.hadoop.mapred.TextInputFormat - file.outputformat org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat - location file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/dest1 - serde: org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe - name: dest1 + Select Operator + expressions: + expr: key + type: string + expr: value + type: string + File Output Operator + directory: /tmp/hive-athusoo/72296119/1324603677.10000.insclause-0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + properties: + name dest1 + serialization.ddl struct dest1 { i32 key, string value} + serialization.format 1 + columns key,value + bucket_count -1 + serialization.lib org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe + file.inputformat org.apache.hadoop.mapred.TextInputFormat + file.outputformat org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + location file:/data/users/athusoo/hadoop_local_ws1/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/dest1 + serde: org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe + name: dest1 Needs Tagging: Path -> Alias: - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/srcbucket + file:/data/users/athusoo/hadoop_local_ws1/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/srcbucket/kv1.txt Path -> Partition: - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/srcbucket + file:/data/users/athusoo/hadoop_local_ws1/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/srcbucket/kv1.txt Partition input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat properties: name srcbucket + bucket_field_name key serialization.ddl struct srcbucket { string key, string value} - serialization.format 1 columns key,value + serialization.format 1 bucket_count 2 serialization.lib org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat - location file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/srcbucket + location file:/data/users/athusoo/hadoop_local_ws1/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/srcbucket serde: org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe name: srcbucket @@ -64,7 +60,7 @@ Move Operator tables: replace: - source: /tmp/hive-njain/594156852/317367185.10000.insclause-0 + source: /tmp/hive-athusoo/72296119/1324603677.10000.insclause-0 table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat @@ -73,38 +69,60 @@ serialization.ddl struct dest1 { i32 key, string value} serialization.format 1 columns key,value - SORTBUCKETCOLSPREFIX TRUE bucket_count -1 serialization.lib org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat - location file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/dest1 + location file:/data/users/athusoo/hadoop_local_ws1/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/dest1 serde: org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe name: dest1 +238 val_238 86 val_86 +311 val_311 +27 val_27 165 val_165 +409 val_409 255 val_255 +278 val_278 +98 val_98 484 val_484 +265 val_265 +193 val_193 +401 val_401 150 val_150 273 val_273 224 val_224 369 val_369 66 val_66 +128 val_128 213 val_213 +146 val_146 406 val_406 +429 val_429 374 val_374 152 val_152 +469 val_469 145 val_145 495 val_495 37 val_37 327 val_327 +281 val_281 277 val_277 +209 val_209 15 val_15 82 val_82 +403 val_403 +166 val_166 417 val_417 +430 val_430 +252 val_252 +292 val_292 219 val_219 +287 val_287 +153 val_153 +193 val_193 338 val_338 446 val_446 459 val_459 @@ -113,105 +131,202 @@ 482 val_482 174 val_174 413 val_413 +494 val_494 +207 val_207 +199 val_199 466 val_466 208 val_208 174 val_174 +399 val_399 396 val_396 +247 val_247 417 val_417 +489 val_489 +162 val_162 +377 val_377 +397 val_397 309 val_309 365 val_365 266 val_266 439 val_439 +342 val_342 367 val_367 325 val_325 167 val_167 +195 val_195 475 val_475 17 val_17 +113 val_113 +155 val_155 +203 val_203 +339 val_339 0 val_0 455 val_455 +128 val_128 +311 val_311 316 val_316 57 val_57 +302 val_302 +205 val_205 149 val_149 +438 val_438 345 val_345 129 val_129 170 val_170 20 val_20 +489 val_489 +157 val_157 378 val_378 +221 val_221 +92 val_92 +111 val_111 +47 val_47 +72 val_72 4 val_4 280 val_280 35 val_35 +427 val_427 277 val_277 208 val_208 356 val_356 +399 val_399 169 val_169 +382 val_382 +498 val_498 125 val_125 +386 val_386 437 val_437 +469 val_469 192 val_192 286 val_286 187 val_187 176 val_176 +54 val_54 459 val_459 51 val_51 138 val_138 103 val_103 239 val_239 213 val_213 +216 val_216 +430 val_430 +278 val_278 176 val_176 +289 val_289 +221 val_221 +65 val_65 318 val_318 332 val_332 +311 val_311 275 val_275 +137 val_137 +241 val_241 +83 val_83 +333 val_333 +180 val_180 284 val_284 +12 val_12 +230 val_230 181 val_181 +67 val_67 260 val_260 404 val_404 +384 val_384 +489 val_489 +353 val_353 +373 val_373 +272 val_272 138 val_138 217 val_217 84 val_84 +348 val_348 466 val_466 +58 val_58 8 val_8 411 val_411 +230 val_230 208 val_208 +348 val_348 24 val_24 +463 val_463 431 val_431 +179 val_179 172 val_172 42 val_42 129 val_129 158 val_158 +119 val_119 +496 val_496 0 val_0 +322 val_322 +197 val_197 468 val_468 +393 val_393 +454 val_454 +100 val_100 +298 val_298 +199 val_199 +191 val_191 +418 val_418 +96 val_96 26 val_26 165 val_165 327 val_327 +230 val_230 +205 val_205 +120 val_120 +131 val_131 51 val_51 404 val_404 +43 val_43 +436 val_436 156 val_156 +469 val_469 468 val_468 +308 val_308 95 val_95 196 val_196 288 val_288 +481 val_481 457 val_457 +98 val_98 282 val_282 +197 val_197 187 val_187 318 val_318 318 val_318 +409 val_409 +470 val_470 +137 val_137 369 val_369 316 val_316 169 val_169 413 val_413 +85 val_85 77 val_77 0 val_0 +490 val_490 +87 val_87 +364 val_364 +179 val_179 118 val_118 134 val_134 +395 val_395 282 val_282 138 val_138 +238 val_238 419 val_419 15 val_15 118 val_118 +72 val_72 +90 val_90 307 val_307 19 val_19 435 val_435 +10 val_10 277 val_277 273 val_273 +306 val_306 224 val_224 309 val_309 389 val_389 @@ -219,365 +334,247 @@ 242 val_242 369 val_369 392 val_392 +272 val_272 +331 val_331 +401 val_401 242 val_242 +452 val_452 +177 val_177 226 val_226 +5 val_5 497 val_497 402 val_402 396 val_396 +317 val_317 +395 val_395 +58 val_58 35 val_35 336 val_336 95 val_95 11 val_11 +168 val_168 +34 val_34 +229 val_229 233 val_233 143 val_143 +472 val_472 +322 val_322 +498 val_498 +160 val_160 +195 val_195 42 val_42 321 val_321 +430 val_430 +119 val_119 +489 val_489 +458 val_458 +78 val_78 +76 val_76 +41 val_41 +223 val_223 +492 val_492 149 val_149 +449 val_449 +218 val_218 228 val_228 138 val_138 453 val_453 +30 val_30 +209 val_209 64 val_64 468 val_468 +76 val_76 +74 val_74 +342 val_342 +69 val_69 +230 val_230 33 val_33 +368 val_368 103 val_103 +296 val_296 +113 val_113 +216 val_216 367 val_367 +344 val_344 167 val_167 +274 val_274 219 val_219 239 val_239 +485 val_485 116 val_116 +223 val_223 +256 val_256 +263 val_263 +70 val_70 +487 val_487 480 val_480 +401 val_401 288 val_288 +191 val_191 +5 val_5 244 val_244 +438 val_438 +128 val_128 +467 val_467 +432 val_432 202 val_202 316 val_316 +229 val_229 +469 val_469 +463 val_463 280 val_280 2 val_2 35 val_35 +283 val_283 +331 val_331 235 val_235 80 val_80 44 val_44 +193 val_193 321 val_321 +335 val_335 +104 val_104 466 val_466 +366 val_366 +175 val_175 +403 val_403 +483 val_483 53 val_53 105 val_105 257 val_257 406 val_406 +409 val_409 190 val_190 406 val_406 +401 val_401 114 val_114 +258 val_258 +90 val_90 +203 val_203 262 val_262 +348 val_348 424 val_424 +12 val_12 396 val_396 +201 val_201 217 val_217 +164 val_164 431 val_431 +454 val_454 +478 val_478 +298 val_298 125 val_125 431 val_431 +164 val_164 424 val_424 187 val_187 +382 val_382 +5 val_5 +70 val_70 +397 val_397 480 val_480 291 val_291 24 val_24 +351 val_351 255 val_255 +104 val_104 +70 val_70 163 val_163 +438 val_438 +119 val_119 +414 val_414 200 val_200 491 val_491 237 val_237 439 val_439 +360 val_360 248 val_248 479 val_479 305 val_305 417 val_417 +199 val_199 444 val_444 +120 val_120 +429 val_429 169 val_169 +443 val_443 323 val_323 325 val_325 277 val_277 +230 val_230 +478 val_478 178 val_178 468 val_468 310 val_310 +317 val_317 +333 val_333 493 val_493 460 val_460 +207 val_207 +249 val_249 +265 val_265 480 val_480 +83 val_83 136 val_136 +353 val_353 172 val_172 +214 val_214 462 val_462 233 val_233 406 val_406 +133 val_133 +175 val_175 189 val_189 +454 val_454 +375 val_375 +401 val_401 +421 val_421 +407 val_407 +384 val_384 +256 val_256 26 val_26 134 val_134 +67 val_67 +384 val_384 +379 val_379 +18 val_18 462 val_462 +492 val_492 +100 val_100 +298 val_298 +9 val_9 341 val_341 +498 val_498 +146 val_146 +458 val_458 +362 val_362 +186 val_186 +285 val_285 +348 val_348 167 val_167 +18 val_18 273 val_273 183 val_183 +281 val_281 +344 val_344 97 val_97 +469 val_469 +315 val_315 84 val_84 28 val_28 37 val_37 448 val_448 152 val_152 +348 val_348 307 val_307 194 val_194 +414 val_414 477 val_477 222 val_222 +126 val_126 +90 val_90 169 val_169 +403 val_403 400 val_400 200 val_200 97 val_97 -291 val_292 -62 val_63 -271 val_272 -217 val_218 -167 val_168 -468 val_469 -413 val_414 -455 val_456 -231 val_232 -448 val_449 -246 val_247 -440 val_441 -31 val_32 -147 val_148 -428 val_429 -273 val_274 -356 val_357 -217 val_218 -33 val_34 -35 val_36 -402 val_403 -226 val_227 -15 val_16 -110 val_111 -343 val_344 -275 val_276 -0 val_1 -293 val_294 -240 val_241 -286 val_287 -408 val_409 -477 val_478 -455 val_456 -99 val_100 -482 val_483 -367 val_368 -59 val_60 -48 val_49 -424 val_425 -226 val_227 -488 val_489 -349 val_350 -11 val_12 -161 val_162 -123 val_124 -402 val_403 -468 val_469 -314 val_315 -497 val_498 -224 val_225 -4 val_5 -277 val_278 -206 val_207 -101 val_102 -114 val_115 -239 val_240 -389 val_390 -114 val_115 -235 val_236 -390 val_391 -165 val_166 -264 val_265 -196 val_197 -20 val_21 -11 val_12 -129 val_130 -257 val_258 -71 val_72 -453 val_454 -82 val_83 -86 val_87 -323 val_324 -222 val_223 -42 val_43 -411 val_412 -338 val_339 -68 val_69 -185 val_186 -147 val_148 -473 val_474 -376 val_377 -347 val_348 -213 val_214 -291 val_292 -426 val_427 -132 val_133 -446 val_447 -121 val_122 -284 val_285 -129 val_130 -246 val_247 -491 val_492 -369 val_370 -262 val_263 -390 val_391 -352 val_353 -226 val_227 -349 val_350 -480 val_481 -497 val_498 -480 val_481 -475 val_476 -183 val_184 -15 val_16 -318 val_319 -149 val_150 -392 val_393 -77 val_78 -275 val_276 -174 val_175 -114 val_115 -161 val_162 -75 val_76 -264 val_265 -48 val_49 -336 val_337 -390 val_391 -484 val_485 -189 val_190 -305 val_306 -367 val_368 -6 val_7 -260 val_261 -2 val_3 -349 val_350 -170 val_171 -51 val_52 -363 val_364 -118 val_119 -310 val_311 -19 val_20 -80 val_81 -121 val_122 -239 val_240 -358 val_359 -11 val_12 -143 val_144 -341 val_342 -105 val_106 -42 val_43 -363 val_364 -347 val_348 -77 val_78 -293 val_294 -20 val_21 -40 val_41 -8 val_9 -310 val_311 -206 val_207 -381 val_382 -404 val_405 -206 val_207 -385 val_386 -259 val_260 -226 val_227 -262 val_263 -303 val_304 -260 val_261 -404 val_405 -116 val_117 -42 val_43 -303 val_304 -93 val_94 -277 val_278 -406 val_407 -451 val_452 -53 val_54 -132 val_133 -192 val_193 -356 val_357 -352 val_353 -330 val_331 -138 val_139 -257 val_258 -174 val_175 -295 val_296 -105 val_106 -204 val_205 -439 val_440 -374 val_375 -457 val_458 -93 val_94 -196 val_197 -437 val_438 -398 val_399 -35 val_36 -334 val_335 -435 val_436 -60 val_61 -442 val_443 -404 val_405 -446 val_447 -0 val_1 -268 val_269 -491 val_492 -392 val_393 -295 val_296 -93 val_94 -439 val_440 -156 val_157 -341 val_342 -118 val_119 -172 val_173 -244 val_245 -6 val_7 -284 val_285 -136 val_137 -349 val_350 -462 val_463 -123 val_124 -408 val_409 -152 val_153 -310 val_311 -486 val_487 -152 val_153 -378 val_379 -97 val_98 -495 val_496 -385 val_386 -40 val_41 -389 val_390 -259 val_260 -97 val_98 -125 val_126 -178 val_179 -415 val_416 -156 val_157 -228 val_229 -363 val_364 -22 val_23 -248 val_249 -459 val_460 -402 val_403 -62 val_63 -244 val_245 -46 val_47 -341 val_342 -215 val_216 -134 val_135 -196 val_197 -431 val_432 -82 val_83 -440 val_441 -309 val_310 -161 val_162 -468 val_469 -152 val_153 -138 val_139 -161 val_162 -91 val_92 -118 val_119 -178 val_179 -457 val_458 -242 val_243 -35 val_36 -244 val_245 Index: src/contrib/hive/ql/src/test/results/clientpositive/groupby2_map.q.out =================================================================== --- src/contrib/hive/ql/src/test/results/clientpositive/groupby2_map.q.out (revision 0) +++ src/contrib/hive/ql/src/test/results/clientpositive/groupby2_map.q.out (revision 0) @@ -0,0 +1,110 @@ +ABSTRACT SYNTAX TREE: + (TOK_QUERY (TOK_FROM (TOK_TABREF src)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB dest1)) (TOK_SELECT (TOK_SELEXPR (TOK_FUNCTION substr (TOK_COLREF src key) 0 1)) (TOK_SELEXPR (TOK_FUNCTIONDI count (TOK_FUNCTION substr (TOK_COLREF src value) 4))) (TOK_SELEXPR (TOK_FUNCTION concat (TOK_FUNCTION substr (TOK_COLREF src key) 0 1) (TOK_FUNCTION sum (TOK_FUNCTION substr (TOK_COLREF src value) 4))))) (TOK_GROUPBY (TOK_FUNCTION substr (TOK_COLREF src key) 0 1)))) + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Alias -> Map Operator Tree: + src + Group By Operator + + expr: count(DISTINCT substr(value, 4)) + expr: sum(UDFToDouble(substr(value, 4))) + keys: + expr: substr(key, 0, 1) + type: string + expr: substr(value, 4) + type: string + mode: hash + Reduce Output Operator + key expressions: + expr: 0 + type: string + expr: 1 + type: string + # partition fields: 2147483647 + tag: -1 + value expressions: + expr: 2 + type: bigint + expr: 3 + type: double + Reduce Operator Tree: + Group By Operator + + expr: count(DISTINCT KEY.1) + expr: sum(VALUE.1) + keys: + expr: KEY.0 + type: string + mode: partial2 + File Output Operator + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.mapred.SequenceFileOutputFormat + name: binary_table + + Stage: Stage-2 + Map Reduce + Alias -> Map Operator Tree: + /tmp/hive-athusoo/822454399/434423100.10001 + Reduce Output Operator + key expressions: + expr: 0 + type: string + # partition fields: 1 + tag: -1 + value expressions: + expr: 1 + type: bigint + expr: 2 + type: double + Reduce Operator Tree: + Group By Operator + + expr: count(VALUE.0) + expr: sum(VALUE.1) + keys: + expr: KEY.0 + type: string + mode: unknown + Select Operator + expressions: + expr: 0 + type: string + expr: 1 + type: bigint + expr: concat(0, UDFToString(2)) + type: string + File Output Operator + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe + name: dest1 + + Stage: Stage-0 + Move Operator + tables: + replace: + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe + name: dest1 + +0 1 00.0 +1 71 116414.0 +2 69 225571.0 +3 62 332004.0 +4 74 452763.0 +5 6 5397.0 +6 5 6398.0 +7 6 7735.0 +8 8 8762.0 +9 7 91047.0 Index: src/contrib/hive/ql/src/test/results/clientpositive/inputddl3.q.out =================================================================== --- src/contrib/hive/ql/src/test/results/clientpositive/inputddl3.q.out (revision 711472) +++ src/contrib/hive/ql/src/test/results/clientpositive/inputddl3.q.out (working copy) @@ -1,5 +1,5 @@ ABSTRACT SYNTAX TREE: - (TOK_CREATETABLE INPUTDDL3 (TOK_TABCOLLIST (TOK_TABCOL key TOK_INT) (TOK_TABCOL value TOK_STRING)) (TOK_TABLEROWFORMAT (TOK_TABLEROWFORMATFIELD '\t'))) + (TOK_CREATETABLE INPUTDDL3 (TOK_TABCOLLIST (TOK_TABCOL key TOK_INT) (TOK_TABCOL value TOK_STRING)) (TOK_TABLEROWFORMAT (TOK_TABLEROWFORMATFIELD '\t')) TOK_TBLTEXTFILE) STAGE DEPENDENCIES: Stage-0 is a root stage Index: src/contrib/hive/ql/src/test/results/clientpositive/sample6.q.out =================================================================== --- src/contrib/hive/ql/src/test/results/clientpositive/sample6.q.out (revision 711472) +++ src/contrib/hive/ql/src/test/results/clientpositive/sample6.q.out (working copy) @@ -1,5 +1,5 @@ ABSTRACT SYNTAX TREE: - (TOK_QUERY (TOK_FROM (TOK_TABREF srcbucket (TOK_TABLESAMPLE 1 4 key) s)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB dest1)) (TOK_SELECT (TOK_SELEXPR (TOK_ALLCOLREF s))))) + (TOK_QUERY (TOK_FROM (TOK_TABREF srcbucket (TOK_TABLESAMPLE 1 4 (TOK_COLREF key)) s)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB dest1)) (TOK_SELECT (TOK_SELEXPR (TOK_ALLCOLREF s))))) STAGE DEPENDENCIES: Stage-1 is a root stage @@ -13,7 +13,7 @@ Filter Operator predicate: expr: (((default_sample_hashfn(key) & 2147483647) % 4) = 0) - type: Boolean + type: boolean Select Operator expressions: expr: key @@ -21,7 +21,7 @@ expr: value type: string File Output Operator - directory: /tmp/hive-njain/391691394/413823912.10000.insclause-0 + directory: /tmp/hive-athusoo/246711872/1816884394.10000.insclause-0 table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat @@ -30,33 +30,33 @@ serialization.ddl struct dest1 { i32 key, string value} serialization.format 1 columns key,value - SORTBUCKETCOLSPREFIX TRUE bucket_count -1 serialization.lib org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat - location file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/dest1 + location file:/data/users/athusoo/hadoop_local_ws1/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/dest1 serde: org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe name: dest1 Needs Tagging: Path -> Alias: - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/srcbucket + file:/data/users/athusoo/hadoop_local_ws1/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/srcbucket/kv1.txt Path -> Partition: - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/srcbucket + file:/data/users/athusoo/hadoop_local_ws1/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/srcbucket/kv1.txt Partition input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat properties: name srcbucket + bucket_field_name key serialization.ddl struct srcbucket { string key, string value} - serialization.format 1 columns key,value + serialization.format 1 bucket_count 2 serialization.lib org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat - location file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/srcbucket + location file:/data/users/athusoo/hadoop_local_ws1/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/srcbucket serde: org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe name: srcbucket @@ -64,7 +64,7 @@ Move Operator tables: replace: - source: /tmp/hive-njain/391691394/413823912.10000.insclause-0 + source: /tmp/hive-athusoo/246711872/1816884394.10000.insclause-0 table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat @@ -73,12 +73,11 @@ serialization.ddl struct dest1 { i32 key, string value} serialization.format 1 columns key,value - SORTBUCKETCOLSPREFIX TRUE bucket_count -1 serialization.lib org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat - location file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/dest1 + location file:/data/users/athusoo/hadoop_local_ws1/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/dest1 serde: org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe name: dest1 @@ -201,138 +200,3 @@ 477 val_477 169 val_169 400 val_400 -62 val_63 -271 val_272 -217 val_218 -455 val_456 -231 val_232 -448 val_449 -246 val_247 -440 val_441 -147 val_148 -356 val_357 -217 val_218 -33 val_34 -15 val_16 -110 val_111 -275 val_276 -0 val_1 -293 val_294 -286 val_287 -408 val_409 -477 val_478 -455 val_456 -99 val_100 -367 val_368 -59 val_60 -48 val_49 -488 val_489 -349 val_350 -11 val_12 -161 val_162 -224 val_225 -4 val_5 -206 val_207 -114 val_115 -239 val_240 -389 val_390 -114 val_115 -235 val_236 -165 val_166 -264 val_265 -11 val_12 -129 val_130 -257 val_258 -323 val_324 -411 val_412 -338 val_339 -147 val_148 -473 val_474 -213 val_214 -426 val_427 -132 val_133 -121 val_122 -129 val_130 -246 val_247 -491 val_492 -352 val_353 -349 val_350 -480 val_481 -480 val_481 -183 val_184 -15 val_16 -392 val_393 -77 val_78 -275 val_276 -114 val_115 -161 val_162 -264 val_265 -48 val_49 -484 val_485 -305 val_306 -367 val_368 -260 val_261 -349 val_350 -51 val_52 -363 val_364 -118 val_119 -19 val_20 -80 val_81 -121 val_122 -239 val_240 -11 val_12 -143 val_144 -341 val_342 -363 val_364 -77 val_78 -293 val_294 -40 val_41 -8 val_9 -206 val_207 -381 val_382 -404 val_405 -206 val_207 -385 val_386 -260 val_261 -404 val_405 -451 val_452 -132 val_133 -356 val_357 -352 val_353 -330 val_331 -257 val_258 -374 val_375 -437 val_438 -334 val_335 -404 val_405 -0 val_1 -268 val_269 -491 val_492 -392 val_393 -341 val_342 -118 val_119 -172 val_173 -136 val_137 -349 val_350 -462 val_463 -408 val_409 -378 val_379 -495 val_496 -385 val_386 -40 val_41 -389 val_390 -125 val_126 -415 val_416 -228 val_229 -363 val_364 -22 val_23 -459 val_460 -62 val_63 -341 val_342 -440 val_441 -309 val_310 -161 val_162 -161 val_162 -91 val_92 -118 val_119 -242 val_243 Index: src/contrib/hive/ql/src/test/results/clientpositive/groupby4_map.q.out =================================================================== --- src/contrib/hive/ql/src/test/results/clientpositive/groupby4_map.q.out (revision 0) +++ src/contrib/hive/ql/src/test/results/clientpositive/groupby4_map.q.out (revision 0) @@ -0,0 +1,50 @@ +ABSTRACT SYNTAX TREE: + (TOK_QUERY (TOK_FROM (TOK_TABREF src)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB dest1)) (TOK_SELECT (TOK_SELEXPR (TOK_FUNCTION count 1))))) + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Alias -> Map Operator Tree: + src + Group By Operator + + expr: count(1) + mode: hash + Reduce Output Operator + # partition fields: -1 + tag: -1 + value expressions: + expr: 0 + type: bigint + Reduce Operator Tree: + Group By Operator + + expr: count(VALUE.0) + mode: unknown + Select Operator + expressions: + expr: 0 + type: bigint + File Output Operator + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe + name: dest1 + + Stage: Stage-0 + Move Operator + tables: + replace: + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe + name: dest1 + + +500 Index: src/contrib/hive/ql/src/test/results/clientpositive/inputddl7.q.out =================================================================== --- src/contrib/hive/ql/src/test/results/clientpositive/inputddl7.q.out (revision 0) +++ src/contrib/hive/ql/src/test/results/clientpositive/inputddl7.q.out (revision 0) @@ -0,0 +1,20 @@ +500 +500 +500 +500 +name string +Detailed Table Information: +Table(tableName:t1,dbName:default,owner:athusoo,createTime:1225764040,lastAccessTime:0,retention:0,sd:StorageDescriptor(cols:[FieldSchema(name:name,type:string,comment:null)],location:file:/data/users/athusoo/hadoop_local_ws1/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/t1,inputFormat:org.apache.hadoop.mapred.TextInputFormat,outputFormat:org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat,compressed:false,numBuckets:-1,serdeInfo:SerDeInfo(name:null,serializationLib:org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe,parameters:{serialization.format=1}),bucketCols:[],sortCols:[],parameters:{}),partitionKeys:[],parameters:{}) +name string +Detailed Table Information: +Table(tableName:t2,dbName:default,owner:athusoo,createTime:1225764046,lastAccessTime:0,retention:0,sd:StorageDescriptor(cols:[FieldSchema(name:name,type:string,comment:null)],location:file:/data/users/athusoo/hadoop_local_ws1/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/t2,inputFormat:org.apache.hadoop.mapred.SequenceFileInputFormat,outputFormat:org.apache.hadoop.mapred.SequenceFileOutputFormat,compressed:false,numBuckets:-1,serdeInfo:SerDeInfo(name:null,serializationLib:org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe,parameters:{serialization.format=1}),bucketCols:[],sortCols:[],parameters:{}),partitionKeys:[],parameters:{}) +name string +ds string + +Detailed Partition Information: +Partition(values:[2008-04-09],dbName:default,tableName:t3,createTime:0,lastAccessTime:0,sd:StorageDescriptor(cols:[FieldSchema(name:name,type:string,comment:null)],location:file:/data/users/athusoo/hadoop_local_ws1/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/t3/ds=2008-04-09,inputFormat:org.apache.hadoop.mapred.TextInputFormat,outputFormat:org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat,compressed:false,numBuckets:-1,serdeInfo:SerDeInfo(name:null,serializationLib:org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe,parameters:{serialization.format=1}),bucketCols:[],sortCols:[],parameters:{}),parameters:{}) +name string +ds string + +Detailed Partition Information: +Partition(values:[2008-04-09],dbName:default,tableName:t4,createTime:0,lastAccessTime:0,sd:StorageDescriptor(cols:[FieldSchema(name:name,type:string,comment:null)],location:file:/data/users/athusoo/hadoop_local_ws1/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/t4/ds=2008-04-09,inputFormat:org.apache.hadoop.mapred.SequenceFileInputFormat,outputFormat:org.apache.hadoop.mapred.SequenceFileOutputFormat,compressed:false,numBuckets:-1,serdeInfo:SerDeInfo(name:null,serializationLib:org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe,parameters:{serialization.format=1}),bucketCols:[],sortCols:[],parameters:{}),parameters:{}) Index: src/contrib/hive/ql/src/test/results/clientpositive/notable_alias1.q.out =================================================================== --- src/contrib/hive/ql/src/test/results/clientpositive/notable_alias1.q.out (revision 0) +++ src/contrib/hive/ql/src/test/results/clientpositive/notable_alias1.q.out (revision 0) @@ -0,0 +1,144 @@ +ABSTRACT SYNTAX TREE: + (TOK_QUERY (TOK_FROM (TOK_TABREF src)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB dest1)) (TOK_SELECT (TOK_SELEXPR '1234') (TOK_SELEXPR (TOK_COLREF key)) (TOK_SELEXPR (TOK_FUNCTION count 1))) (TOK_WHERE (< (TOK_COLREF src key) 100)) (TOK_GROUPBY (TOK_COLREF key)))) + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Alias -> Map Operator Tree: + src + Filter Operator + predicate: + expr: (key < 100) + type: boolean + Reduce Output Operator + key expressions: + expr: key + type: string + # partition fields: -1 + tag: -1 + value expressions: + expr: 1 + type: int + Reduce Operator Tree: + Group By Operator + + expr: count(VALUE.0) + keys: + expr: KEY.0 + type: string + mode: partial1 + File Output Operator + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.mapred.SequenceFileOutputFormat + name: binary_table + + Stage: Stage-2 + Map Reduce + Alias -> Map Operator Tree: + /tmp/hive-athusoo/17687212/895842287.10001 + Reduce Output Operator + key expressions: + expr: 0 + type: string + # partition fields: 1 + tag: -1 + value expressions: + expr: 1 + type: bigint + Reduce Operator Tree: + Group By Operator + + expr: count(VALUE.0) + keys: + expr: KEY.0 + type: string + mode: unknown + Select Operator + expressions: + expr: '1234' + type: string + expr: 0 + type: string + expr: 1 + type: bigint + File Output Operator + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe + name: dest1 + + Stage: Stage-0 + Move Operator + tables: + replace: + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe + name: dest1 + + +1234 0 3 +1234 10 1 +1234 11 1 +1234 12 2 +1234 15 2 +1234 17 1 +1234 18 2 +1234 19 1 +1234 2 1 +1234 20 1 +1234 24 2 +1234 26 2 +1234 27 1 +1234 28 1 +1234 30 1 +1234 33 1 +1234 34 1 +1234 35 3 +1234 37 2 +1234 4 1 +1234 41 1 +1234 42 2 +1234 43 1 +1234 44 1 +1234 47 1 +1234 5 3 +1234 51 2 +1234 53 1 +1234 54 1 +1234 57 1 +1234 58 2 +1234 64 1 +1234 65 1 +1234 66 1 +1234 67 2 +1234 69 1 +1234 70 3 +1234 72 2 +1234 74 1 +1234 76 2 +1234 77 1 +1234 78 1 +1234 8 1 +1234 80 1 +1234 82 1 +1234 83 2 +1234 84 2 +1234 85 1 +1234 86 1 +1234 87 1 +1234 9 1 +1234 90 3 +1234 92 1 +1234 95 2 +1234 96 1 +1234 97 2 +1234 98 2 Index: src/contrib/hive/ql/src/test/results/clientpositive/input4.q.out =================================================================== --- src/contrib/hive/ql/src/test/results/clientpositive/input4.q.out (revision 711472) +++ src/contrib/hive/ql/src/test/results/clientpositive/input4.q.out (working copy) @@ -8,8 +8,8 @@ STAGE PLANS: Stage: Stage-0 Copy - source: file:/home/njain/workspace/hadoop-0.17/src/contrib/hive/data/files/kv1.txt - destination: file:/tmp/hive-njain/-478939701 + source: file:/data/users/athusoo/hadoop_local_ws1/trunk/VENDOR/hadoop-0.17/src/contrib/hive/data/files/kv1.txt + destination: file:/tmp/hive-athusoo/-1031452266 Stage: Stage-1 Move Operator Index: src/contrib/hive/ql/src/test/results/clientpositive/join5.q.out =================================================================== --- src/contrib/hive/ql/src/test/results/clientpositive/join5.q.out (revision 711472) +++ src/contrib/hive/ql/src/test/results/clientpositive/join5.q.out (working copy) @@ -13,7 +13,7 @@ Filter Operator predicate: expr: ((key > 15) and (key < 25)) - type: Boolean + type: boolean Select Operator expressions: expr: key @@ -35,7 +35,7 @@ Filter Operator predicate: expr: ((key > 10) and (key < 20)) - type: Boolean + type: boolean Select Operator expressions: expr: key Index: src/contrib/hive/ql/src/test/results/clientpositive/input6.q.out =================================================================== --- src/contrib/hive/ql/src/test/results/clientpositive/input6.q.out (revision 711472) +++ src/contrib/hive/ql/src/test/results/clientpositive/input6.q.out (working copy) @@ -13,7 +13,7 @@ Filter Operator predicate: expr: key is null - type: Boolean + type: boolean Select Operator expressions: expr: key Index: src/contrib/hive/ql/src/test/results/clientpositive/input_testxpath2.q.out =================================================================== --- src/contrib/hive/ql/src/test/results/clientpositive/input_testxpath2.q.out (revision 711472) +++ src/contrib/hive/ql/src/test/results/clientpositive/input_testxpath2.q.out (working copy) @@ -13,7 +13,7 @@ Filter Operator predicate: expr: (lint is not null and not mstringstring is null) - type: Boolean + type: boolean Select Operator expressions: expr: size(lint) Index: src/contrib/hive/ql/src/test/results/clientpositive/join7.q.out =================================================================== --- src/contrib/hive/ql/src/test/results/clientpositive/join7.q.out (revision 711472) +++ src/contrib/hive/ql/src/test/results/clientpositive/join7.q.out (working copy) @@ -13,7 +13,7 @@ Filter Operator predicate: expr: ((key > 15) and (key < 25)) - type: Boolean + type: boolean Select Operator expressions: expr: key @@ -35,7 +35,7 @@ Filter Operator predicate: expr: ((key > 10) and (key < 20)) - type: Boolean + type: boolean Select Operator expressions: expr: key @@ -57,7 +57,7 @@ Filter Operator predicate: expr: ((key > 20) and (key < 25)) - type: Boolean + type: boolean Select Operator expressions: expr: key Index: src/contrib/hive/ql/src/test/results/clientpositive/input8.q.out =================================================================== --- src/contrib/hive/ql/src/test/results/clientpositive/input8.q.out (revision 711472) +++ src/contrib/hive/ql/src/test/results/clientpositive/input8.q.out (working copy) @@ -17,7 +17,7 @@ expr: (UDFToDouble(key) - null) type: double expr: (null + null) - type: Byte + type: tinyint File Output Operator table: input format: org.apache.hadoop.mapred.TextInputFormat Index: src/contrib/hive/ql/src/test/results/clientpositive/union.q.out =================================================================== --- src/contrib/hive/ql/src/test/results/clientpositive/union.q.out (revision 711472) +++ src/contrib/hive/ql/src/test/results/clientpositive/union.q.out (working copy) @@ -13,7 +13,7 @@ Filter Operator predicate: expr: (key < 100) - type: Boolean + type: boolean Select Operator expressions: expr: key @@ -34,7 +34,7 @@ Filter Operator predicate: expr: (key > 100) - type: Boolean + type: boolean Select Operator expressions: expr: key Index: src/contrib/hive/ql/src/test/results/clientpositive/join9.q.out =================================================================== --- src/contrib/hive/ql/src/test/results/clientpositive/join9.q.out (revision 711472) +++ src/contrib/hive/ql/src/test/results/clientpositive/join9.q.out (working copy) @@ -39,10 +39,10 @@ type: string Needs Tagging: Path -> Alias: - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/src - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/srcpart/ds=2008-04-08/hr=12 + file:/data/users/athusoo/hadoop_local_ws1/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/src + file:/data/users/athusoo/hadoop_local_ws1/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/srcpart/ds=2008-04-08/hr=12 Path -> Partition: - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/src + file:/data/users/athusoo/hadoop_local_ws1/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/src Partition input format: org.apache.hadoop.mapred.TextInputFormat @@ -56,10 +56,10 @@ serialization.lib org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat - location file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/src + location file:/data/users/athusoo/hadoop_local_ws1/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/src serde: org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe name: src - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/srcpart/ds=2008-04-08/hr=12 + file:/data/users/athusoo/hadoop_local_ws1/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/srcpart/ds=2008-04-08/hr=12 Partition partition values: ds 2008-04-08 @@ -77,7 +77,7 @@ serialization.lib org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat - location file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/srcpart + location file:/data/users/athusoo/hadoop_local_ws1/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/srcpart serde: org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe name: srcpart Reduce Operator Tree: @@ -90,7 +90,7 @@ Filter Operator predicate: expr: ((2 = '2008-04-08') and (3 = '12')) - type: Boolean + type: boolean Select Operator expressions: expr: 0 @@ -98,7 +98,7 @@ expr: 5 type: string File Output Operator - directory: /tmp/hive-njain/465936460/211979279.10000.insclause-0 + directory: /tmp/hive-athusoo/1438963112/465759079.10000.insclause-0 table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat @@ -107,12 +107,11 @@ serialization.ddl struct dest1 { i32 key, string value} serialization.format 1 columns key,value - SORTBUCKETCOLSPREFIX TRUE bucket_count -1 serialization.lib org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat - location file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/dest1 + location file:/data/users/athusoo/hadoop_local_ws1/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/dest1 serde: org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe name: dest1 @@ -120,7 +119,7 @@ Move Operator tables: replace: - source: /tmp/hive-njain/465936460/211979279.10000.insclause-0 + source: /tmp/hive-athusoo/1438963112/465759079.10000.insclause-0 table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat @@ -129,12 +128,11 @@ serialization.ddl struct dest1 { i32 key, string value} serialization.format 1 columns key,value - SORTBUCKETCOLSPREFIX TRUE bucket_count -1 serialization.lib org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat - location file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/dest1 + location file:/data/users/athusoo/hadoop_local_ws1/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/dest1 serde: org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe name: dest1 Index: src/contrib/hive/ql/src/test/results/clientpositive/input12.q.out =================================================================== --- src/contrib/hive/ql/src/test/results/clientpositive/input12.q.out (revision 711472) +++ src/contrib/hive/ql/src/test/results/clientpositive/input12.q.out (working copy) @@ -13,7 +13,7 @@ Filter Operator predicate: expr: (key < 100) - type: Boolean + type: boolean Select Operator expressions: expr: key @@ -29,7 +29,7 @@ Filter Operator predicate: expr: ((key >= 100) and (key < 200)) - type: Boolean + type: boolean Select Operator expressions: expr: key @@ -45,7 +45,7 @@ Filter Operator predicate: expr: (key >= 200) - type: Boolean + type: boolean Select Operator expressions: expr: key @@ -272,314 +272,314 @@ 194 val_194 126 val_126 169 val_169 -238 NULL 2008-04-08 12 -311 NULL 2008-04-08 12 -409 NULL 2008-04-08 12 -255 NULL 2008-04-08 12 -278 NULL 2008-04-08 12 -484 NULL 2008-04-08 12 -265 NULL 2008-04-08 12 -401 NULL 2008-04-08 12 -273 NULL 2008-04-08 12 -224 NULL 2008-04-08 12 -369 NULL 2008-04-08 12 -213 NULL 2008-04-08 12 -406 NULL 2008-04-08 12 -429 NULL 2008-04-08 12 -374 NULL 2008-04-08 12 -469 NULL 2008-04-08 12 -495 NULL 2008-04-08 12 -327 NULL 2008-04-08 12 -281 NULL 2008-04-08 12 -277 NULL 2008-04-08 12 -209 NULL 2008-04-08 12 -403 NULL 2008-04-08 12 -417 NULL 2008-04-08 12 -430 NULL 2008-04-08 12 -252 NULL 2008-04-08 12 -292 NULL 2008-04-08 12 -219 NULL 2008-04-08 12 -287 NULL 2008-04-08 12 -338 NULL 2008-04-08 12 -446 NULL 2008-04-08 12 -459 NULL 2008-04-08 12 -394 NULL 2008-04-08 12 -237 NULL 2008-04-08 12 -482 NULL 2008-04-08 12 -413 NULL 2008-04-08 12 -494 NULL 2008-04-08 12 -207 NULL 2008-04-08 12 -466 NULL 2008-04-08 12 -208 NULL 2008-04-08 12 -399 NULL 2008-04-08 12 -396 NULL 2008-04-08 12 -247 NULL 2008-04-08 12 -417 NULL 2008-04-08 12 -489 NULL 2008-04-08 12 -377 NULL 2008-04-08 12 -397 NULL 2008-04-08 12 -309 NULL 2008-04-08 12 -365 NULL 2008-04-08 12 -266 NULL 2008-04-08 12 -439 NULL 2008-04-08 12 -342 NULL 2008-04-08 12 -367 NULL 2008-04-08 12 -325 NULL 2008-04-08 12 -475 NULL 2008-04-08 12 -203 NULL 2008-04-08 12 -339 NULL 2008-04-08 12 -455 NULL 2008-04-08 12 -311 NULL 2008-04-08 12 -316 NULL 2008-04-08 12 -302 NULL 2008-04-08 12 -205 NULL 2008-04-08 12 -438 NULL 2008-04-08 12 -345 NULL 2008-04-08 12 -489 NULL 2008-04-08 12 -378 NULL 2008-04-08 12 -221 NULL 2008-04-08 12 -280 NULL 2008-04-08 12 -427 NULL 2008-04-08 12 -277 NULL 2008-04-08 12 -208 NULL 2008-04-08 12 -356 NULL 2008-04-08 12 -399 NULL 2008-04-08 12 -382 NULL 2008-04-08 12 -498 NULL 2008-04-08 12 -386 NULL 2008-04-08 12 -437 NULL 2008-04-08 12 -469 NULL 2008-04-08 12 -286 NULL 2008-04-08 12 -459 NULL 2008-04-08 12 -239 NULL 2008-04-08 12 -213 NULL 2008-04-08 12 -216 NULL 2008-04-08 12 -430 NULL 2008-04-08 12 -278 NULL 2008-04-08 12 -289 NULL 2008-04-08 12 -221 NULL 2008-04-08 12 -318 NULL 2008-04-08 12 -332 NULL 2008-04-08 12 -311 NULL 2008-04-08 12 -275 NULL 2008-04-08 12 -241 NULL 2008-04-08 12 -333 NULL 2008-04-08 12 -284 NULL 2008-04-08 12 -230 NULL 2008-04-08 12 -260 NULL 2008-04-08 12 -404 NULL 2008-04-08 12 -384 NULL 2008-04-08 12 -489 NULL 2008-04-08 12 -353 NULL 2008-04-08 12 -373 NULL 2008-04-08 12 -272 NULL 2008-04-08 12 -217 NULL 2008-04-08 12 -348 NULL 2008-04-08 12 -466 NULL 2008-04-08 12 -411 NULL 2008-04-08 12 -230 NULL 2008-04-08 12 -208 NULL 2008-04-08 12 -348 NULL 2008-04-08 12 -463 NULL 2008-04-08 12 -431 NULL 2008-04-08 12 -496 NULL 2008-04-08 12 -322 NULL 2008-04-08 12 -468 NULL 2008-04-08 12 -393 NULL 2008-04-08 12 -454 NULL 2008-04-08 12 -298 NULL 2008-04-08 12 -418 NULL 2008-04-08 12 -327 NULL 2008-04-08 12 -230 NULL 2008-04-08 12 -205 NULL 2008-04-08 12 -404 NULL 2008-04-08 12 -436 NULL 2008-04-08 12 -469 NULL 2008-04-08 12 -468 NULL 2008-04-08 12 -308 NULL 2008-04-08 12 -288 NULL 2008-04-08 12 -481 NULL 2008-04-08 12 -457 NULL 2008-04-08 12 -282 NULL 2008-04-08 12 -318 NULL 2008-04-08 12 -318 NULL 2008-04-08 12 -409 NULL 2008-04-08 12 -470 NULL 2008-04-08 12 -369 NULL 2008-04-08 12 -316 NULL 2008-04-08 12 -413 NULL 2008-04-08 12 -490 NULL 2008-04-08 12 -364 NULL 2008-04-08 12 -395 NULL 2008-04-08 12 -282 NULL 2008-04-08 12 -238 NULL 2008-04-08 12 -419 NULL 2008-04-08 12 -307 NULL 2008-04-08 12 -435 NULL 2008-04-08 12 -277 NULL 2008-04-08 12 -273 NULL 2008-04-08 12 -306 NULL 2008-04-08 12 -224 NULL 2008-04-08 12 -309 NULL 2008-04-08 12 -389 NULL 2008-04-08 12 -327 NULL 2008-04-08 12 -242 NULL 2008-04-08 12 -369 NULL 2008-04-08 12 -392 NULL 2008-04-08 12 -272 NULL 2008-04-08 12 -331 NULL 2008-04-08 12 -401 NULL 2008-04-08 12 -242 NULL 2008-04-08 12 -452 NULL 2008-04-08 12 -226 NULL 2008-04-08 12 -497 NULL 2008-04-08 12 -402 NULL 2008-04-08 12 -396 NULL 2008-04-08 12 -317 NULL 2008-04-08 12 -395 NULL 2008-04-08 12 -336 NULL 2008-04-08 12 -229 NULL 2008-04-08 12 -233 NULL 2008-04-08 12 -472 NULL 2008-04-08 12 -322 NULL 2008-04-08 12 -498 NULL 2008-04-08 12 -321 NULL 2008-04-08 12 -430 NULL 2008-04-08 12 -489 NULL 2008-04-08 12 -458 NULL 2008-04-08 12 -223 NULL 2008-04-08 12 -492 NULL 2008-04-08 12 -449 NULL 2008-04-08 12 -218 NULL 2008-04-08 12 -228 NULL 2008-04-08 12 -453 NULL 2008-04-08 12 -209 NULL 2008-04-08 12 -468 NULL 2008-04-08 12 -342 NULL 2008-04-08 12 -230 NULL 2008-04-08 12 -368 NULL 2008-04-08 12 -296 NULL 2008-04-08 12 -216 NULL 2008-04-08 12 -367 NULL 2008-04-08 12 -344 NULL 2008-04-08 12 -274 NULL 2008-04-08 12 -219 NULL 2008-04-08 12 -239 NULL 2008-04-08 12 -485 NULL 2008-04-08 12 -223 NULL 2008-04-08 12 -256 NULL 2008-04-08 12 -263 NULL 2008-04-08 12 -487 NULL 2008-04-08 12 -480 NULL 2008-04-08 12 -401 NULL 2008-04-08 12 -288 NULL 2008-04-08 12 -244 NULL 2008-04-08 12 -438 NULL 2008-04-08 12 -467 NULL 2008-04-08 12 -432 NULL 2008-04-08 12 -202 NULL 2008-04-08 12 -316 NULL 2008-04-08 12 -229 NULL 2008-04-08 12 -469 NULL 2008-04-08 12 -463 NULL 2008-04-08 12 -280 NULL 2008-04-08 12 -283 NULL 2008-04-08 12 -331 NULL 2008-04-08 12 -235 NULL 2008-04-08 12 -321 NULL 2008-04-08 12 -335 NULL 2008-04-08 12 -466 NULL 2008-04-08 12 -366 NULL 2008-04-08 12 -403 NULL 2008-04-08 12 -483 NULL 2008-04-08 12 -257 NULL 2008-04-08 12 -406 NULL 2008-04-08 12 -409 NULL 2008-04-08 12 -406 NULL 2008-04-08 12 -401 NULL 2008-04-08 12 -258 NULL 2008-04-08 12 -203 NULL 2008-04-08 12 -262 NULL 2008-04-08 12 -348 NULL 2008-04-08 12 -424 NULL 2008-04-08 12 -396 NULL 2008-04-08 12 -201 NULL 2008-04-08 12 -217 NULL 2008-04-08 12 -431 NULL 2008-04-08 12 -454 NULL 2008-04-08 12 -478 NULL 2008-04-08 12 -298 NULL 2008-04-08 12 -431 NULL 2008-04-08 12 -424 NULL 2008-04-08 12 -382 NULL 2008-04-08 12 -397 NULL 2008-04-08 12 -480 NULL 2008-04-08 12 -291 NULL 2008-04-08 12 -351 NULL 2008-04-08 12 -255 NULL 2008-04-08 12 -438 NULL 2008-04-08 12 -414 NULL 2008-04-08 12 -200 NULL 2008-04-08 12 -491 NULL 2008-04-08 12 -237 NULL 2008-04-08 12 -439 NULL 2008-04-08 12 -360 NULL 2008-04-08 12 -248 NULL 2008-04-08 12 -479 NULL 2008-04-08 12 -305 NULL 2008-04-08 12 -417 NULL 2008-04-08 12 -444 NULL 2008-04-08 12 -429 NULL 2008-04-08 12 -443 NULL 2008-04-08 12 -323 NULL 2008-04-08 12 -325 NULL 2008-04-08 12 -277 NULL 2008-04-08 12 -230 NULL 2008-04-08 12 -478 NULL 2008-04-08 12 -468 NULL 2008-04-08 12 -310 NULL 2008-04-08 12 -317 NULL 2008-04-08 12 -333 NULL 2008-04-08 12 -493 NULL 2008-04-08 12 -460 NULL 2008-04-08 12 -207 NULL 2008-04-08 12 -249 NULL 2008-04-08 12 -265 NULL 2008-04-08 12 -480 NULL 2008-04-08 12 -353 NULL 2008-04-08 12 -214 NULL 2008-04-08 12 -462 NULL 2008-04-08 12 -233 NULL 2008-04-08 12 -406 NULL 2008-04-08 12 -454 NULL 2008-04-08 12 -375 NULL 2008-04-08 12 -401 NULL 2008-04-08 12 -421 NULL 2008-04-08 12 -407 NULL 2008-04-08 12 -384 NULL 2008-04-08 12 -256 NULL 2008-04-08 12 -384 NULL 2008-04-08 12 -379 NULL 2008-04-08 12 -462 NULL 2008-04-08 12 -492 NULL 2008-04-08 12 -298 NULL 2008-04-08 12 -341 NULL 2008-04-08 12 -498 NULL 2008-04-08 12 -458 NULL 2008-04-08 12 -362 NULL 2008-04-08 12 -285 NULL 2008-04-08 12 -348 NULL 2008-04-08 12 -273 NULL 2008-04-08 12 -281 NULL 2008-04-08 12 -344 NULL 2008-04-08 12 -469 NULL 2008-04-08 12 -315 NULL 2008-04-08 12 -448 NULL 2008-04-08 12 -348 NULL 2008-04-08 12 -307 NULL 2008-04-08 12 -414 NULL 2008-04-08 12 -477 NULL 2008-04-08 12 -222 NULL 2008-04-08 12 -403 NULL 2008-04-08 12 -400 NULL 2008-04-08 12 -200 NULL 2008-04-08 12 +238 2008-04-08 12 +311 2008-04-08 12 +409 2008-04-08 12 +255 2008-04-08 12 +278 2008-04-08 12 +484 2008-04-08 12 +265 2008-04-08 12 +401 2008-04-08 12 +273 2008-04-08 12 +224 2008-04-08 12 +369 2008-04-08 12 +213 2008-04-08 12 +406 2008-04-08 12 +429 2008-04-08 12 +374 2008-04-08 12 +469 2008-04-08 12 +495 2008-04-08 12 +327 2008-04-08 12 +281 2008-04-08 12 +277 2008-04-08 12 +209 2008-04-08 12 +403 2008-04-08 12 +417 2008-04-08 12 +430 2008-04-08 12 +252 2008-04-08 12 +292 2008-04-08 12 +219 2008-04-08 12 +287 2008-04-08 12 +338 2008-04-08 12 +446 2008-04-08 12 +459 2008-04-08 12 +394 2008-04-08 12 +237 2008-04-08 12 +482 2008-04-08 12 +413 2008-04-08 12 +494 2008-04-08 12 +207 2008-04-08 12 +466 2008-04-08 12 +208 2008-04-08 12 +399 2008-04-08 12 +396 2008-04-08 12 +247 2008-04-08 12 +417 2008-04-08 12 +489 2008-04-08 12 +377 2008-04-08 12 +397 2008-04-08 12 +309 2008-04-08 12 +365 2008-04-08 12 +266 2008-04-08 12 +439 2008-04-08 12 +342 2008-04-08 12 +367 2008-04-08 12 +325 2008-04-08 12 +475 2008-04-08 12 +203 2008-04-08 12 +339 2008-04-08 12 +455 2008-04-08 12 +311 2008-04-08 12 +316 2008-04-08 12 +302 2008-04-08 12 +205 2008-04-08 12 +438 2008-04-08 12 +345 2008-04-08 12 +489 2008-04-08 12 +378 2008-04-08 12 +221 2008-04-08 12 +280 2008-04-08 12 +427 2008-04-08 12 +277 2008-04-08 12 +208 2008-04-08 12 +356 2008-04-08 12 +399 2008-04-08 12 +382 2008-04-08 12 +498 2008-04-08 12 +386 2008-04-08 12 +437 2008-04-08 12 +469 2008-04-08 12 +286 2008-04-08 12 +459 2008-04-08 12 +239 2008-04-08 12 +213 2008-04-08 12 +216 2008-04-08 12 +430 2008-04-08 12 +278 2008-04-08 12 +289 2008-04-08 12 +221 2008-04-08 12 +318 2008-04-08 12 +332 2008-04-08 12 +311 2008-04-08 12 +275 2008-04-08 12 +241 2008-04-08 12 +333 2008-04-08 12 +284 2008-04-08 12 +230 2008-04-08 12 +260 2008-04-08 12 +404 2008-04-08 12 +384 2008-04-08 12 +489 2008-04-08 12 +353 2008-04-08 12 +373 2008-04-08 12 +272 2008-04-08 12 +217 2008-04-08 12 +348 2008-04-08 12 +466 2008-04-08 12 +411 2008-04-08 12 +230 2008-04-08 12 +208 2008-04-08 12 +348 2008-04-08 12 +463 2008-04-08 12 +431 2008-04-08 12 +496 2008-04-08 12 +322 2008-04-08 12 +468 2008-04-08 12 +393 2008-04-08 12 +454 2008-04-08 12 +298 2008-04-08 12 +418 2008-04-08 12 +327 2008-04-08 12 +230 2008-04-08 12 +205 2008-04-08 12 +404 2008-04-08 12 +436 2008-04-08 12 +469 2008-04-08 12 +468 2008-04-08 12 +308 2008-04-08 12 +288 2008-04-08 12 +481 2008-04-08 12 +457 2008-04-08 12 +282 2008-04-08 12 +318 2008-04-08 12 +318 2008-04-08 12 +409 2008-04-08 12 +470 2008-04-08 12 +369 2008-04-08 12 +316 2008-04-08 12 +413 2008-04-08 12 +490 2008-04-08 12 +364 2008-04-08 12 +395 2008-04-08 12 +282 2008-04-08 12 +238 2008-04-08 12 +419 2008-04-08 12 +307 2008-04-08 12 +435 2008-04-08 12 +277 2008-04-08 12 +273 2008-04-08 12 +306 2008-04-08 12 +224 2008-04-08 12 +309 2008-04-08 12 +389 2008-04-08 12 +327 2008-04-08 12 +242 2008-04-08 12 +369 2008-04-08 12 +392 2008-04-08 12 +272 2008-04-08 12 +331 2008-04-08 12 +401 2008-04-08 12 +242 2008-04-08 12 +452 2008-04-08 12 +226 2008-04-08 12 +497 2008-04-08 12 +402 2008-04-08 12 +396 2008-04-08 12 +317 2008-04-08 12 +395 2008-04-08 12 +336 2008-04-08 12 +229 2008-04-08 12 +233 2008-04-08 12 +472 2008-04-08 12 +322 2008-04-08 12 +498 2008-04-08 12 +321 2008-04-08 12 +430 2008-04-08 12 +489 2008-04-08 12 +458 2008-04-08 12 +223 2008-04-08 12 +492 2008-04-08 12 +449 2008-04-08 12 +218 2008-04-08 12 +228 2008-04-08 12 +453 2008-04-08 12 +209 2008-04-08 12 +468 2008-04-08 12 +342 2008-04-08 12 +230 2008-04-08 12 +368 2008-04-08 12 +296 2008-04-08 12 +216 2008-04-08 12 +367 2008-04-08 12 +344 2008-04-08 12 +274 2008-04-08 12 +219 2008-04-08 12 +239 2008-04-08 12 +485 2008-04-08 12 +223 2008-04-08 12 +256 2008-04-08 12 +263 2008-04-08 12 +487 2008-04-08 12 +480 2008-04-08 12 +401 2008-04-08 12 +288 2008-04-08 12 +244 2008-04-08 12 +438 2008-04-08 12 +467 2008-04-08 12 +432 2008-04-08 12 +202 2008-04-08 12 +316 2008-04-08 12 +229 2008-04-08 12 +469 2008-04-08 12 +463 2008-04-08 12 +280 2008-04-08 12 +283 2008-04-08 12 +331 2008-04-08 12 +235 2008-04-08 12 +321 2008-04-08 12 +335 2008-04-08 12 +466 2008-04-08 12 +366 2008-04-08 12 +403 2008-04-08 12 +483 2008-04-08 12 +257 2008-04-08 12 +406 2008-04-08 12 +409 2008-04-08 12 +406 2008-04-08 12 +401 2008-04-08 12 +258 2008-04-08 12 +203 2008-04-08 12 +262 2008-04-08 12 +348 2008-04-08 12 +424 2008-04-08 12 +396 2008-04-08 12 +201 2008-04-08 12 +217 2008-04-08 12 +431 2008-04-08 12 +454 2008-04-08 12 +478 2008-04-08 12 +298 2008-04-08 12 +431 2008-04-08 12 +424 2008-04-08 12 +382 2008-04-08 12 +397 2008-04-08 12 +480 2008-04-08 12 +291 2008-04-08 12 +351 2008-04-08 12 +255 2008-04-08 12 +438 2008-04-08 12 +414 2008-04-08 12 +200 2008-04-08 12 +491 2008-04-08 12 +237 2008-04-08 12 +439 2008-04-08 12 +360 2008-04-08 12 +248 2008-04-08 12 +479 2008-04-08 12 +305 2008-04-08 12 +417 2008-04-08 12 +444 2008-04-08 12 +429 2008-04-08 12 +443 2008-04-08 12 +323 2008-04-08 12 +325 2008-04-08 12 +277 2008-04-08 12 +230 2008-04-08 12 +478 2008-04-08 12 +468 2008-04-08 12 +310 2008-04-08 12 +317 2008-04-08 12 +333 2008-04-08 12 +493 2008-04-08 12 +460 2008-04-08 12 +207 2008-04-08 12 +249 2008-04-08 12 +265 2008-04-08 12 +480 2008-04-08 12 +353 2008-04-08 12 +214 2008-04-08 12 +462 2008-04-08 12 +233 2008-04-08 12 +406 2008-04-08 12 +454 2008-04-08 12 +375 2008-04-08 12 +401 2008-04-08 12 +421 2008-04-08 12 +407 2008-04-08 12 +384 2008-04-08 12 +256 2008-04-08 12 +384 2008-04-08 12 +379 2008-04-08 12 +462 2008-04-08 12 +492 2008-04-08 12 +298 2008-04-08 12 +341 2008-04-08 12 +498 2008-04-08 12 +458 2008-04-08 12 +362 2008-04-08 12 +285 2008-04-08 12 +348 2008-04-08 12 +273 2008-04-08 12 +281 2008-04-08 12 +344 2008-04-08 12 +469 2008-04-08 12 +315 2008-04-08 12 +448 2008-04-08 12 +348 2008-04-08 12 +307 2008-04-08 12 +414 2008-04-08 12 +477 2008-04-08 12 +222 2008-04-08 12 +403 2008-04-08 12 +400 2008-04-08 12 +200 2008-04-08 12 Index: src/contrib/hive/ql/src/test/results/clientpositive/input14.q.out =================================================================== --- src/contrib/hive/ql/src/test/results/clientpositive/input14.q.out (revision 711472) +++ src/contrib/hive/ql/src/test/results/clientpositive/input14.q.out (working copy) @@ -1,5 +1,5 @@ ABSTRACT SYNTAX TREE: - (TOK_QUERY (TOK_FROM (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF src)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TRANSFORM (TOK_COLLIST (TOK_COLREF src key) (TOK_COLREF src value)) (TOK_ALIASLIST tkey tvalue) '/bin/cat'))) (TOK_CLUSTERBY tkey))) tmap)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB dest1)) (TOK_SELECT (TOK_SELEXPR (TOK_COLREF tmap tkey)) (TOK_SELEXPR (TOK_COLREF tmap tvalue))) (TOK_WHERE (< (TOK_COLREF tmap tkey) 100)))) + (TOK_QUERY (TOK_FROM (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF src)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TRANSFORM (TOK_EXPLIST (TOK_COLREF src key) (TOK_COLREF src value)) (TOK_ALIASLIST tkey tvalue) '/bin/cat'))) (TOK_CLUSTERBY tkey))) tmap)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB dest1)) (TOK_SELECT (TOK_SELEXPR (TOK_COLREF tmap tkey)) (TOK_SELEXPR (TOK_COLREF tmap tvalue))) (TOK_WHERE (< (TOK_COLREF tmap tkey) 100)))) STAGE DEPENDENCIES: Stage-1 is a root stage @@ -37,7 +37,7 @@ Filter Operator predicate: expr: (0 < 100) - type: Boolean + type: boolean Select Operator expressions: expr: 0 Index: src/contrib/hive/ql/src/test/results/clientpositive/input_part1.q.out =================================================================== --- src/contrib/hive/ql/src/test/results/clientpositive/input_part1.q.out (revision 711472) +++ src/contrib/hive/ql/src/test/results/clientpositive/input_part1.q.out (working copy) @@ -13,7 +13,7 @@ Filter Operator predicate: expr: (((key < 100) and (ds = '2008-04-08')) and (hr = '12')) - type: Boolean + type: boolean Select Operator expressions: expr: key @@ -25,7 +25,7 @@ expr: ds type: string File Output Operator - directory: /tmp/hive-njain/13222008/293248514.10000.insclause-0 + directory: /tmp/hive-athusoo/222427714/1556500954.10000.insclause-0 table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat @@ -34,19 +34,18 @@ serialization.ddl struct dest1 { i32 key, string value, string hr, string ds} serialization.format 1 columns key,value,hr,ds - SORTBUCKETCOLSPREFIX TRUE bucket_count -1 serialization.lib org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat - location file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/dest1 + location file:/data/users/athusoo/hadoop_local_ws1/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/dest1 serde: org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe name: dest1 Needs Tagging: Path -> Alias: - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/srcpart/ds=2008-04-08/hr=12 + file:/data/users/athusoo/hadoop_local_ws1/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/srcpart/ds=2008-04-08/hr=12 Path -> Partition: - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/srcpart/ds=2008-04-08/hr=12 + file:/data/users/athusoo/hadoop_local_ws1/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/srcpart/ds=2008-04-08/hr=12 Partition partition values: ds 2008-04-08 @@ -64,7 +63,7 @@ serialization.lib org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat - location file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/srcpart + location file:/data/users/athusoo/hadoop_local_ws1/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/srcpart serde: org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe name: srcpart @@ -72,7 +71,7 @@ Move Operator tables: replace: - source: /tmp/hive-njain/13222008/293248514.10000.insclause-0 + source: /tmp/hive-athusoo/222427714/1556500954.10000.insclause-0 table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat @@ -81,12 +80,11 @@ serialization.ddl struct dest1 { i32 key, string value, string hr, string ds} serialization.format 1 columns key,value,hr,ds - SORTBUCKETCOLSPREFIX TRUE bucket_count -1 serialization.lib org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat - location file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/dest1 + location file:/data/users/athusoo/hadoop_local_ws1/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/dest1 serde: org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe name: dest1 Index: src/contrib/hive/ql/src/test/results/clientpositive/input_part3.q.out =================================================================== --- src/contrib/hive/ql/src/test/results/clientpositive/input_part3.q.out (revision 0) +++ src/contrib/hive/ql/src/test/results/clientpositive/input_part3.q.out (revision 0) @@ -0,0 +1,512 @@ +ABSTRACT SYNTAX TREE: + (TOK_QUERY (TOK_FROM (TOK_TABREF SRCPART x)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_ALLCOLREF x))) (TOK_WHERE (and (= (TOK_COLREF x ds) '2008-04-08') (= (TOK_COLREF x hr) 11))))) + +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + + +238 val_238 2008-04-08 11 +86 val_86 2008-04-08 11 +311 val_311 2008-04-08 11 +27 val_27 2008-04-08 11 +165 val_165 2008-04-08 11 +409 val_409 2008-04-08 11 +255 val_255 2008-04-08 11 +278 val_278 2008-04-08 11 +98 val_98 2008-04-08 11 +484 val_484 2008-04-08 11 +265 val_265 2008-04-08 11 +193 val_193 2008-04-08 11 +401 val_401 2008-04-08 11 +150 val_150 2008-04-08 11 +273 val_273 2008-04-08 11 +224 val_224 2008-04-08 11 +369 val_369 2008-04-08 11 +66 val_66 2008-04-08 11 +128 val_128 2008-04-08 11 +213 val_213 2008-04-08 11 +146 val_146 2008-04-08 11 +406 val_406 2008-04-08 11 +429 val_429 2008-04-08 11 +374 val_374 2008-04-08 11 +152 val_152 2008-04-08 11 +469 val_469 2008-04-08 11 +145 val_145 2008-04-08 11 +495 val_495 2008-04-08 11 +37 val_37 2008-04-08 11 +327 val_327 2008-04-08 11 +281 val_281 2008-04-08 11 +277 val_277 2008-04-08 11 +209 val_209 2008-04-08 11 +15 val_15 2008-04-08 11 +82 val_82 2008-04-08 11 +403 val_403 2008-04-08 11 +166 val_166 2008-04-08 11 +417 val_417 2008-04-08 11 +430 val_430 2008-04-08 11 +252 val_252 2008-04-08 11 +292 val_292 2008-04-08 11 +219 val_219 2008-04-08 11 +287 val_287 2008-04-08 11 +153 val_153 2008-04-08 11 +193 val_193 2008-04-08 11 +338 val_338 2008-04-08 11 +446 val_446 2008-04-08 11 +459 val_459 2008-04-08 11 +394 val_394 2008-04-08 11 +237 val_237 2008-04-08 11 +482 val_482 2008-04-08 11 +174 val_174 2008-04-08 11 +413 val_413 2008-04-08 11 +494 val_494 2008-04-08 11 +207 val_207 2008-04-08 11 +199 val_199 2008-04-08 11 +466 val_466 2008-04-08 11 +208 val_208 2008-04-08 11 +174 val_174 2008-04-08 11 +399 val_399 2008-04-08 11 +396 val_396 2008-04-08 11 +247 val_247 2008-04-08 11 +417 val_417 2008-04-08 11 +489 val_489 2008-04-08 11 +162 val_162 2008-04-08 11 +377 val_377 2008-04-08 11 +397 val_397 2008-04-08 11 +309 val_309 2008-04-08 11 +365 val_365 2008-04-08 11 +266 val_266 2008-04-08 11 +439 val_439 2008-04-08 11 +342 val_342 2008-04-08 11 +367 val_367 2008-04-08 11 +325 val_325 2008-04-08 11 +167 val_167 2008-04-08 11 +195 val_195 2008-04-08 11 +475 val_475 2008-04-08 11 +17 val_17 2008-04-08 11 +113 val_113 2008-04-08 11 +155 val_155 2008-04-08 11 +203 val_203 2008-04-08 11 +339 val_339 2008-04-08 11 +0 val_0 2008-04-08 11 +455 val_455 2008-04-08 11 +128 val_128 2008-04-08 11 +311 val_311 2008-04-08 11 +316 val_316 2008-04-08 11 +57 val_57 2008-04-08 11 +302 val_302 2008-04-08 11 +205 val_205 2008-04-08 11 +149 val_149 2008-04-08 11 +438 val_438 2008-04-08 11 +345 val_345 2008-04-08 11 +129 val_129 2008-04-08 11 +170 val_170 2008-04-08 11 +20 val_20 2008-04-08 11 +489 val_489 2008-04-08 11 +157 val_157 2008-04-08 11 +378 val_378 2008-04-08 11 +221 val_221 2008-04-08 11 +92 val_92 2008-04-08 11 +111 val_111 2008-04-08 11 +47 val_47 2008-04-08 11 +72 val_72 2008-04-08 11 +4 val_4 2008-04-08 11 +280 val_280 2008-04-08 11 +35 val_35 2008-04-08 11 +427 val_427 2008-04-08 11 +277 val_277 2008-04-08 11 +208 val_208 2008-04-08 11 +356 val_356 2008-04-08 11 +399 val_399 2008-04-08 11 +169 val_169 2008-04-08 11 +382 val_382 2008-04-08 11 +498 val_498 2008-04-08 11 +125 val_125 2008-04-08 11 +386 val_386 2008-04-08 11 +437 val_437 2008-04-08 11 +469 val_469 2008-04-08 11 +192 val_192 2008-04-08 11 +286 val_286 2008-04-08 11 +187 val_187 2008-04-08 11 +176 val_176 2008-04-08 11 +54 val_54 2008-04-08 11 +459 val_459 2008-04-08 11 +51 val_51 2008-04-08 11 +138 val_138 2008-04-08 11 +103 val_103 2008-04-08 11 +239 val_239 2008-04-08 11 +213 val_213 2008-04-08 11 +216 val_216 2008-04-08 11 +430 val_430 2008-04-08 11 +278 val_278 2008-04-08 11 +176 val_176 2008-04-08 11 +289 val_289 2008-04-08 11 +221 val_221 2008-04-08 11 +65 val_65 2008-04-08 11 +318 val_318 2008-04-08 11 +332 val_332 2008-04-08 11 +311 val_311 2008-04-08 11 +275 val_275 2008-04-08 11 +137 val_137 2008-04-08 11 +241 val_241 2008-04-08 11 +83 val_83 2008-04-08 11 +333 val_333 2008-04-08 11 +180 val_180 2008-04-08 11 +284 val_284 2008-04-08 11 +12 val_12 2008-04-08 11 +230 val_230 2008-04-08 11 +181 val_181 2008-04-08 11 +67 val_67 2008-04-08 11 +260 val_260 2008-04-08 11 +404 val_404 2008-04-08 11 +384 val_384 2008-04-08 11 +489 val_489 2008-04-08 11 +353 val_353 2008-04-08 11 +373 val_373 2008-04-08 11 +272 val_272 2008-04-08 11 +138 val_138 2008-04-08 11 +217 val_217 2008-04-08 11 +84 val_84 2008-04-08 11 +348 val_348 2008-04-08 11 +466 val_466 2008-04-08 11 +58 val_58 2008-04-08 11 +8 val_8 2008-04-08 11 +411 val_411 2008-04-08 11 +230 val_230 2008-04-08 11 +208 val_208 2008-04-08 11 +348 val_348 2008-04-08 11 +24 val_24 2008-04-08 11 +463 val_463 2008-04-08 11 +431 val_431 2008-04-08 11 +179 val_179 2008-04-08 11 +172 val_172 2008-04-08 11 +42 val_42 2008-04-08 11 +129 val_129 2008-04-08 11 +158 val_158 2008-04-08 11 +119 val_119 2008-04-08 11 +496 val_496 2008-04-08 11 +0 val_0 2008-04-08 11 +322 val_322 2008-04-08 11 +197 val_197 2008-04-08 11 +468 val_468 2008-04-08 11 +393 val_393 2008-04-08 11 +454 val_454 2008-04-08 11 +100 val_100 2008-04-08 11 +298 val_298 2008-04-08 11 +199 val_199 2008-04-08 11 +191 val_191 2008-04-08 11 +418 val_418 2008-04-08 11 +96 val_96 2008-04-08 11 +26 val_26 2008-04-08 11 +165 val_165 2008-04-08 11 +327 val_327 2008-04-08 11 +230 val_230 2008-04-08 11 +205 val_205 2008-04-08 11 +120 val_120 2008-04-08 11 +131 val_131 2008-04-08 11 +51 val_51 2008-04-08 11 +404 val_404 2008-04-08 11 +43 val_43 2008-04-08 11 +436 val_436 2008-04-08 11 +156 val_156 2008-04-08 11 +469 val_469 2008-04-08 11 +468 val_468 2008-04-08 11 +308 val_308 2008-04-08 11 +95 val_95 2008-04-08 11 +196 val_196 2008-04-08 11 +288 val_288 2008-04-08 11 +481 val_481 2008-04-08 11 +457 val_457 2008-04-08 11 +98 val_98 2008-04-08 11 +282 val_282 2008-04-08 11 +197 val_197 2008-04-08 11 +187 val_187 2008-04-08 11 +318 val_318 2008-04-08 11 +318 val_318 2008-04-08 11 +409 val_409 2008-04-08 11 +470 val_470 2008-04-08 11 +137 val_137 2008-04-08 11 +369 val_369 2008-04-08 11 +316 val_316 2008-04-08 11 +169 val_169 2008-04-08 11 +413 val_413 2008-04-08 11 +85 val_85 2008-04-08 11 +77 val_77 2008-04-08 11 +0 val_0 2008-04-08 11 +490 val_490 2008-04-08 11 +87 val_87 2008-04-08 11 +364 val_364 2008-04-08 11 +179 val_179 2008-04-08 11 +118 val_118 2008-04-08 11 +134 val_134 2008-04-08 11 +395 val_395 2008-04-08 11 +282 val_282 2008-04-08 11 +138 val_138 2008-04-08 11 +238 val_238 2008-04-08 11 +419 val_419 2008-04-08 11 +15 val_15 2008-04-08 11 +118 val_118 2008-04-08 11 +72 val_72 2008-04-08 11 +90 val_90 2008-04-08 11 +307 val_307 2008-04-08 11 +19 val_19 2008-04-08 11 +435 val_435 2008-04-08 11 +10 val_10 2008-04-08 11 +277 val_277 2008-04-08 11 +273 val_273 2008-04-08 11 +306 val_306 2008-04-08 11 +224 val_224 2008-04-08 11 +309 val_309 2008-04-08 11 +389 val_389 2008-04-08 11 +327 val_327 2008-04-08 11 +242 val_242 2008-04-08 11 +369 val_369 2008-04-08 11 +392 val_392 2008-04-08 11 +272 val_272 2008-04-08 11 +331 val_331 2008-04-08 11 +401 val_401 2008-04-08 11 +242 val_242 2008-04-08 11 +452 val_452 2008-04-08 11 +177 val_177 2008-04-08 11 +226 val_226 2008-04-08 11 +5 val_5 2008-04-08 11 +497 val_497 2008-04-08 11 +402 val_402 2008-04-08 11 +396 val_396 2008-04-08 11 +317 val_317 2008-04-08 11 +395 val_395 2008-04-08 11 +58 val_58 2008-04-08 11 +35 val_35 2008-04-08 11 +336 val_336 2008-04-08 11 +95 val_95 2008-04-08 11 +11 val_11 2008-04-08 11 +168 val_168 2008-04-08 11 +34 val_34 2008-04-08 11 +229 val_229 2008-04-08 11 +233 val_233 2008-04-08 11 +143 val_143 2008-04-08 11 +472 val_472 2008-04-08 11 +322 val_322 2008-04-08 11 +498 val_498 2008-04-08 11 +160 val_160 2008-04-08 11 +195 val_195 2008-04-08 11 +42 val_42 2008-04-08 11 +321 val_321 2008-04-08 11 +430 val_430 2008-04-08 11 +119 val_119 2008-04-08 11 +489 val_489 2008-04-08 11 +458 val_458 2008-04-08 11 +78 val_78 2008-04-08 11 +76 val_76 2008-04-08 11 +41 val_41 2008-04-08 11 +223 val_223 2008-04-08 11 +492 val_492 2008-04-08 11 +149 val_149 2008-04-08 11 +449 val_449 2008-04-08 11 +218 val_218 2008-04-08 11 +228 val_228 2008-04-08 11 +138 val_138 2008-04-08 11 +453 val_453 2008-04-08 11 +30 val_30 2008-04-08 11 +209 val_209 2008-04-08 11 +64 val_64 2008-04-08 11 +468 val_468 2008-04-08 11 +76 val_76 2008-04-08 11 +74 val_74 2008-04-08 11 +342 val_342 2008-04-08 11 +69 val_69 2008-04-08 11 +230 val_230 2008-04-08 11 +33 val_33 2008-04-08 11 +368 val_368 2008-04-08 11 +103 val_103 2008-04-08 11 +296 val_296 2008-04-08 11 +113 val_113 2008-04-08 11 +216 val_216 2008-04-08 11 +367 val_367 2008-04-08 11 +344 val_344 2008-04-08 11 +167 val_167 2008-04-08 11 +274 val_274 2008-04-08 11 +219 val_219 2008-04-08 11 +239 val_239 2008-04-08 11 +485 val_485 2008-04-08 11 +116 val_116 2008-04-08 11 +223 val_223 2008-04-08 11 +256 val_256 2008-04-08 11 +263 val_263 2008-04-08 11 +70 val_70 2008-04-08 11 +487 val_487 2008-04-08 11 +480 val_480 2008-04-08 11 +401 val_401 2008-04-08 11 +288 val_288 2008-04-08 11 +191 val_191 2008-04-08 11 +5 val_5 2008-04-08 11 +244 val_244 2008-04-08 11 +438 val_438 2008-04-08 11 +128 val_128 2008-04-08 11 +467 val_467 2008-04-08 11 +432 val_432 2008-04-08 11 +202 val_202 2008-04-08 11 +316 val_316 2008-04-08 11 +229 val_229 2008-04-08 11 +469 val_469 2008-04-08 11 +463 val_463 2008-04-08 11 +280 val_280 2008-04-08 11 +2 val_2 2008-04-08 11 +35 val_35 2008-04-08 11 +283 val_283 2008-04-08 11 +331 val_331 2008-04-08 11 +235 val_235 2008-04-08 11 +80 val_80 2008-04-08 11 +44 val_44 2008-04-08 11 +193 val_193 2008-04-08 11 +321 val_321 2008-04-08 11 +335 val_335 2008-04-08 11 +104 val_104 2008-04-08 11 +466 val_466 2008-04-08 11 +366 val_366 2008-04-08 11 +175 val_175 2008-04-08 11 +403 val_403 2008-04-08 11 +483 val_483 2008-04-08 11 +53 val_53 2008-04-08 11 +105 val_105 2008-04-08 11 +257 val_257 2008-04-08 11 +406 val_406 2008-04-08 11 +409 val_409 2008-04-08 11 +190 val_190 2008-04-08 11 +406 val_406 2008-04-08 11 +401 val_401 2008-04-08 11 +114 val_114 2008-04-08 11 +258 val_258 2008-04-08 11 +90 val_90 2008-04-08 11 +203 val_203 2008-04-08 11 +262 val_262 2008-04-08 11 +348 val_348 2008-04-08 11 +424 val_424 2008-04-08 11 +12 val_12 2008-04-08 11 +396 val_396 2008-04-08 11 +201 val_201 2008-04-08 11 +217 val_217 2008-04-08 11 +164 val_164 2008-04-08 11 +431 val_431 2008-04-08 11 +454 val_454 2008-04-08 11 +478 val_478 2008-04-08 11 +298 val_298 2008-04-08 11 +125 val_125 2008-04-08 11 +431 val_431 2008-04-08 11 +164 val_164 2008-04-08 11 +424 val_424 2008-04-08 11 +187 val_187 2008-04-08 11 +382 val_382 2008-04-08 11 +5 val_5 2008-04-08 11 +70 val_70 2008-04-08 11 +397 val_397 2008-04-08 11 +480 val_480 2008-04-08 11 +291 val_291 2008-04-08 11 +24 val_24 2008-04-08 11 +351 val_351 2008-04-08 11 +255 val_255 2008-04-08 11 +104 val_104 2008-04-08 11 +70 val_70 2008-04-08 11 +163 val_163 2008-04-08 11 +438 val_438 2008-04-08 11 +119 val_119 2008-04-08 11 +414 val_414 2008-04-08 11 +200 val_200 2008-04-08 11 +491 val_491 2008-04-08 11 +237 val_237 2008-04-08 11 +439 val_439 2008-04-08 11 +360 val_360 2008-04-08 11 +248 val_248 2008-04-08 11 +479 val_479 2008-04-08 11 +305 val_305 2008-04-08 11 +417 val_417 2008-04-08 11 +199 val_199 2008-04-08 11 +444 val_444 2008-04-08 11 +120 val_120 2008-04-08 11 +429 val_429 2008-04-08 11 +169 val_169 2008-04-08 11 +443 val_443 2008-04-08 11 +323 val_323 2008-04-08 11 +325 val_325 2008-04-08 11 +277 val_277 2008-04-08 11 +230 val_230 2008-04-08 11 +478 val_478 2008-04-08 11 +178 val_178 2008-04-08 11 +468 val_468 2008-04-08 11 +310 val_310 2008-04-08 11 +317 val_317 2008-04-08 11 +333 val_333 2008-04-08 11 +493 val_493 2008-04-08 11 +460 val_460 2008-04-08 11 +207 val_207 2008-04-08 11 +249 val_249 2008-04-08 11 +265 val_265 2008-04-08 11 +480 val_480 2008-04-08 11 +83 val_83 2008-04-08 11 +136 val_136 2008-04-08 11 +353 val_353 2008-04-08 11 +172 val_172 2008-04-08 11 +214 val_214 2008-04-08 11 +462 val_462 2008-04-08 11 +233 val_233 2008-04-08 11 +406 val_406 2008-04-08 11 +133 val_133 2008-04-08 11 +175 val_175 2008-04-08 11 +189 val_189 2008-04-08 11 +454 val_454 2008-04-08 11 +375 val_375 2008-04-08 11 +401 val_401 2008-04-08 11 +421 val_421 2008-04-08 11 +407 val_407 2008-04-08 11 +384 val_384 2008-04-08 11 +256 val_256 2008-04-08 11 +26 val_26 2008-04-08 11 +134 val_134 2008-04-08 11 +67 val_67 2008-04-08 11 +384 val_384 2008-04-08 11 +379 val_379 2008-04-08 11 +18 val_18 2008-04-08 11 +462 val_462 2008-04-08 11 +492 val_492 2008-04-08 11 +100 val_100 2008-04-08 11 +298 val_298 2008-04-08 11 +9 val_9 2008-04-08 11 +341 val_341 2008-04-08 11 +498 val_498 2008-04-08 11 +146 val_146 2008-04-08 11 +458 val_458 2008-04-08 11 +362 val_362 2008-04-08 11 +186 val_186 2008-04-08 11 +285 val_285 2008-04-08 11 +348 val_348 2008-04-08 11 +167 val_167 2008-04-08 11 +18 val_18 2008-04-08 11 +273 val_273 2008-04-08 11 +183 val_183 2008-04-08 11 +281 val_281 2008-04-08 11 +344 val_344 2008-04-08 11 +97 val_97 2008-04-08 11 +469 val_469 2008-04-08 11 +315 val_315 2008-04-08 11 +84 val_84 2008-04-08 11 +28 val_28 2008-04-08 11 +37 val_37 2008-04-08 11 +448 val_448 2008-04-08 11 +152 val_152 2008-04-08 11 +348 val_348 2008-04-08 11 +307 val_307 2008-04-08 11 +194 val_194 2008-04-08 11 +414 val_414 2008-04-08 11 +477 val_477 2008-04-08 11 +222 val_222 2008-04-08 11 +126 val_126 2008-04-08 11 +90 val_90 2008-04-08 11 +169 val_169 2008-04-08 11 +403 val_403 2008-04-08 11 +400 val_400 2008-04-08 11 +200 val_200 2008-04-08 11 +97 val_97 2008-04-08 11 Index: src/contrib/hive/ql/src/test/results/clientpositive/groupby2.q.out =================================================================== --- src/contrib/hive/ql/src/test/results/clientpositive/groupby2.q.out (revision 711472) +++ src/contrib/hive/ql/src/test/results/clientpositive/groupby2.q.out (working copy) @@ -23,20 +23,21 @@ Group By Operator expr: count(DISTINCT KEY.1) - expr: sum(KEY.1) + expr: sum(UDFToDouble(KEY.1)) keys: expr: KEY.0 type: string mode: partial1 File Output Operator table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.mapred.SequenceFileOutputFormat + name: binary_table Stage: Stage-2 Map Reduce Alias -> Map Operator Tree: - /tmp/hive-njain/307368091/808162418.10001 + /tmp/hive-athusoo/792135840/145798048.10001 Reduce Output Operator key expressions: expr: 0 @@ -45,9 +46,9 @@ tag: -1 value expressions: expr: 1 - type: string + type: bigint expr: 2 - type: string + type: double Reduce Operator Tree: Group By Operator @@ -56,15 +57,15 @@ keys: expr: KEY.0 type: string - mode: partial2 + mode: unknown Select Operator expressions: expr: 0 type: string expr: 1 + type: bigint + expr: concat(0, UDFToString(2)) type: string - expr: concat(0, 2) - type: string File Output Operator table: input format: org.apache.hadoop.mapred.TextInputFormat Index: src/contrib/hive/ql/src/test/results/clientpositive/input_part5.q.out =================================================================== --- src/contrib/hive/ql/src/test/results/clientpositive/input_part5.q.out (revision 0) +++ src/contrib/hive/ql/src/test/results/clientpositive/input_part5.q.out (revision 0) @@ -0,0 +1,204 @@ +ABSTRACT SYNTAX TREE: + (TOK_QUERY (TOK_FROM (TOK_TABREF SRCPART x)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_ALLCOLREF x))) (TOK_WHERE (and (= (TOK_COLREF x ds) '2008-04-08') (< (TOK_COLREF x key) 100))))) + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Alias -> Map Operator Tree: + x + Filter Operator + predicate: + expr: ((ds = '2008-04-08') and (key < 100)) + type: boolean + Select Operator + expressions: + expr: key + type: string + expr: value + type: string + expr: ds + type: string + expr: hr + type: string + File Output Operator + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + + Stage: Stage-0 + Fetch Operator + limit: -1 + + +86 val_86 2008-04-08 11 +27 val_27 2008-04-08 11 +98 val_98 2008-04-08 11 +66 val_66 2008-04-08 11 +37 val_37 2008-04-08 11 +15 val_15 2008-04-08 11 +82 val_82 2008-04-08 11 +17 val_17 2008-04-08 11 +0 val_0 2008-04-08 11 +57 val_57 2008-04-08 11 +20 val_20 2008-04-08 11 +92 val_92 2008-04-08 11 +47 val_47 2008-04-08 11 +72 val_72 2008-04-08 11 +4 val_4 2008-04-08 11 +35 val_35 2008-04-08 11 +54 val_54 2008-04-08 11 +51 val_51 2008-04-08 11 +65 val_65 2008-04-08 11 +83 val_83 2008-04-08 11 +12 val_12 2008-04-08 11 +67 val_67 2008-04-08 11 +84 val_84 2008-04-08 11 +58 val_58 2008-04-08 11 +8 val_8 2008-04-08 11 +24 val_24 2008-04-08 11 +42 val_42 2008-04-08 11 +0 val_0 2008-04-08 11 +96 val_96 2008-04-08 11 +26 val_26 2008-04-08 11 +51 val_51 2008-04-08 11 +43 val_43 2008-04-08 11 +95 val_95 2008-04-08 11 +98 val_98 2008-04-08 11 +85 val_85 2008-04-08 11 +77 val_77 2008-04-08 11 +0 val_0 2008-04-08 11 +87 val_87 2008-04-08 11 +15 val_15 2008-04-08 11 +72 val_72 2008-04-08 11 +90 val_90 2008-04-08 11 +19 val_19 2008-04-08 11 +10 val_10 2008-04-08 11 +5 val_5 2008-04-08 11 +58 val_58 2008-04-08 11 +35 val_35 2008-04-08 11 +95 val_95 2008-04-08 11 +11 val_11 2008-04-08 11 +34 val_34 2008-04-08 11 +42 val_42 2008-04-08 11 +78 val_78 2008-04-08 11 +76 val_76 2008-04-08 11 +41 val_41 2008-04-08 11 +30 val_30 2008-04-08 11 +64 val_64 2008-04-08 11 +76 val_76 2008-04-08 11 +74 val_74 2008-04-08 11 +69 val_69 2008-04-08 11 +33 val_33 2008-04-08 11 +70 val_70 2008-04-08 11 +5 val_5 2008-04-08 11 +2 val_2 2008-04-08 11 +35 val_35 2008-04-08 11 +80 val_80 2008-04-08 11 +44 val_44 2008-04-08 11 +53 val_53 2008-04-08 11 +90 val_90 2008-04-08 11 +12 val_12 2008-04-08 11 +5 val_5 2008-04-08 11 +70 val_70 2008-04-08 11 +24 val_24 2008-04-08 11 +70 val_70 2008-04-08 11 +83 val_83 2008-04-08 11 +26 val_26 2008-04-08 11 +67 val_67 2008-04-08 11 +18 val_18 2008-04-08 11 +9 val_9 2008-04-08 11 +18 val_18 2008-04-08 11 +97 val_97 2008-04-08 11 +84 val_84 2008-04-08 11 +28 val_28 2008-04-08 11 +37 val_37 2008-04-08 11 +90 val_90 2008-04-08 11 +97 val_97 2008-04-08 11 +86 val_86 2008-04-08 12 +27 val_27 2008-04-08 12 +98 val_98 2008-04-08 12 +66 val_66 2008-04-08 12 +37 val_37 2008-04-08 12 +15 val_15 2008-04-08 12 +82 val_82 2008-04-08 12 +17 val_17 2008-04-08 12 +0 val_0 2008-04-08 12 +57 val_57 2008-04-08 12 +20 val_20 2008-04-08 12 +92 val_92 2008-04-08 12 +47 val_47 2008-04-08 12 +72 val_72 2008-04-08 12 +4 val_4 2008-04-08 12 +35 val_35 2008-04-08 12 +54 val_54 2008-04-08 12 +51 val_51 2008-04-08 12 +65 val_65 2008-04-08 12 +83 val_83 2008-04-08 12 +12 val_12 2008-04-08 12 +67 val_67 2008-04-08 12 +84 val_84 2008-04-08 12 +58 val_58 2008-04-08 12 +8 val_8 2008-04-08 12 +24 val_24 2008-04-08 12 +42 val_42 2008-04-08 12 +0 val_0 2008-04-08 12 +96 val_96 2008-04-08 12 +26 val_26 2008-04-08 12 +51 val_51 2008-04-08 12 +43 val_43 2008-04-08 12 +95 val_95 2008-04-08 12 +98 val_98 2008-04-08 12 +85 val_85 2008-04-08 12 +77 val_77 2008-04-08 12 +0 val_0 2008-04-08 12 +87 val_87 2008-04-08 12 +15 val_15 2008-04-08 12 +72 val_72 2008-04-08 12 +90 val_90 2008-04-08 12 +19 val_19 2008-04-08 12 +10 val_10 2008-04-08 12 +5 val_5 2008-04-08 12 +58 val_58 2008-04-08 12 +35 val_35 2008-04-08 12 +95 val_95 2008-04-08 12 +11 val_11 2008-04-08 12 +34 val_34 2008-04-08 12 +42 val_42 2008-04-08 12 +78 val_78 2008-04-08 12 +76 val_76 2008-04-08 12 +41 val_41 2008-04-08 12 +30 val_30 2008-04-08 12 +64 val_64 2008-04-08 12 +76 val_76 2008-04-08 12 +74 val_74 2008-04-08 12 +69 val_69 2008-04-08 12 +33 val_33 2008-04-08 12 +70 val_70 2008-04-08 12 +5 val_5 2008-04-08 12 +2 val_2 2008-04-08 12 +35 val_35 2008-04-08 12 +80 val_80 2008-04-08 12 +44 val_44 2008-04-08 12 +53 val_53 2008-04-08 12 +90 val_90 2008-04-08 12 +12 val_12 2008-04-08 12 +5 val_5 2008-04-08 12 +70 val_70 2008-04-08 12 +24 val_24 2008-04-08 12 +70 val_70 2008-04-08 12 +83 val_83 2008-04-08 12 +26 val_26 2008-04-08 12 +67 val_67 2008-04-08 12 +18 val_18 2008-04-08 12 +9 val_9 2008-04-08 12 +18 val_18 2008-04-08 12 +97 val_97 2008-04-08 12 +84 val_84 2008-04-08 12 +28 val_28 2008-04-08 12 +37 val_37 2008-04-08 12 +90 val_90 2008-04-08 12 +97 val_97 2008-04-08 12 Index: src/contrib/hive/ql/src/test/results/clientpositive/groupby4.q.out =================================================================== --- src/contrib/hive/ql/src/test/results/clientpositive/groupby4.q.out (revision 711472) +++ src/contrib/hive/ql/src/test/results/clientpositive/groupby4.q.out (working copy) @@ -25,13 +25,14 @@ mode: partial1 File Output Operator table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.mapred.SequenceFileOutputFormat + name: binary_table Stage: Stage-2 Map Reduce Alias -> Map Operator Tree: - /tmp/hive-njain/1561965178/525265780.10001 + /tmp/hive-athusoo/122153787/1194275869.10001 Reduce Output Operator key expressions: expr: 0 @@ -43,7 +44,7 @@ keys: expr: KEY.0 type: string - mode: partial2 + mode: unknown Select Operator expressions: expr: 0 Index: src/contrib/hive/ql/src/test/results/clientpositive/groupby6.q.out =================================================================== --- src/contrib/hive/ql/src/test/results/clientpositive/groupby6.q.out (revision 711472) +++ src/contrib/hive/ql/src/test/results/clientpositive/groupby6.q.out (working copy) @@ -25,13 +25,14 @@ mode: partial1 File Output Operator table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.mapred.SequenceFileOutputFormat + name: binary_table Stage: Stage-2 Map Reduce Alias -> Map Operator Tree: - /tmp/hive-njain/256745338/35530060.10001 + /tmp/hive-athusoo/719393145/1229165586.10001 Reduce Output Operator key expressions: expr: 0 @@ -43,7 +44,7 @@ keys: expr: KEY.0 type: string - mode: partial2 + mode: unknown Select Operator expressions: expr: 0 Index: src/contrib/hive/ql/src/test/results/clientpositive/input1_limit.q.out =================================================================== --- src/contrib/hive/ql/src/test/results/clientpositive/input1_limit.q.out (revision 711472) +++ src/contrib/hive/ql/src/test/results/clientpositive/input1_limit.q.out (working copy) @@ -13,7 +13,7 @@ Filter Operator predicate: expr: (key < 100) - type: Boolean + type: boolean Select Operator expressions: expr: key @@ -30,7 +30,7 @@ Filter Operator predicate: expr: (key < 100) - type: Boolean + type: boolean Select Operator expressions: expr: key Index: src/contrib/hive/ql/src/test/results/clientpositive/input2_limit.q.out =================================================================== --- src/contrib/hive/ql/src/test/results/clientpositive/input2_limit.q.out (revision 711472) +++ src/contrib/hive/ql/src/test/results/clientpositive/input2_limit.q.out (working copy) @@ -13,7 +13,7 @@ Filter Operator predicate: expr: (key < 300) - type: Boolean + type: boolean Select Operator expressions: expr: key Index: src/contrib/hive/ql/src/test/results/clientpositive/case_sensitivity.q.out =================================================================== --- src/contrib/hive/ql/src/test/results/clientpositive/case_sensitivity.q.out (revision 711472) +++ src/contrib/hive/ql/src/test/results/clientpositive/case_sensitivity.q.out (working copy) @@ -13,7 +13,7 @@ Filter Operator predicate: expr: (lint[0] > 0) - type: Boolean + type: boolean Select Operator expressions: expr: lint[1] Index: src/contrib/hive/ql/src/test/results/clientpositive/scriptfile1.q.out =================================================================== --- src/contrib/hive/ql/src/test/results/clientpositive/scriptfile1.q.out (revision 0) +++ src/contrib/hive/ql/src/test/results/clientpositive/scriptfile1.q.out (revision 0) @@ -0,0 +1,9 @@ +10 val_10 +100 val_100 +100 val_100 +103 val_103 +103 val_103 +104 val_104 +104 val_104 +105 val_105 +310 val_310 Index: src/contrib/hive/ql/src/test/results/clientpositive/sample1.q.out =================================================================== --- src/contrib/hive/ql/src/test/results/clientpositive/sample1.q.out (revision 711472) +++ src/contrib/hive/ql/src/test/results/clientpositive/sample1.q.out (working copy) @@ -1,5 +1,5 @@ ABSTRACT SYNTAX TREE: - (TOK_QUERY (TOK_FROM (TOK_TABREF srcpart (TOK_TABLESAMPLE 1 1) s)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB dest1)) (TOK_SELECT (TOK_SELEXPR (TOK_ALLCOLREF s))) (TOK_WHERE (and (= (TOK_COLREF s ds) '2008-04-08') (= (TOK_COLREF s hr) '11'))))) + (TOK_QUERY (TOK_FROM (TOK_TABREF srcpart (TOK_TABLESAMPLE 1 1 (TOK_FUNCTION rand)) s)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB dest1)) (TOK_SELECT (TOK_SELEXPR (TOK_ALLCOLREF s))) (TOK_WHERE (and (= (TOK_COLREF s ds) '2008-04-08') (= (TOK_COLREF s hr) '11'))))) STAGE DEPENDENCIES: Stage-1 is a root stage @@ -12,41 +12,44 @@ s Filter Operator predicate: - expr: ((ds = '2008-04-08') and (hr = '11')) - type: Boolean - Select Operator - expressions: - expr: key - type: string - expr: value - type: string - expr: ds - type: string - expr: hr - type: string - File Output Operator - directory: /tmp/hive-njain/1799446246/1449752317.10000.insclause-0 - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat - properties: - name dest1 - serialization.ddl struct dest1 { i32 key, string value, string dt, string hr} - serialization.format 1 - columns key,value,dt,hr - SORTBUCKETCOLSPREFIX TRUE - bucket_count -1 - serialization.lib org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe - file.inputformat org.apache.hadoop.mapred.TextInputFormat - file.outputformat org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat - location file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/dest1 - serde: org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe - name: dest1 + expr: (((default_sample_hashfn(rand()) & 2147483647) % 1) = 0) + type: boolean + Filter Operator + predicate: + expr: ((ds = '2008-04-08') and (hr = '11')) + type: boolean + Select Operator + expressions: + expr: key + type: string + expr: value + type: string + expr: ds + type: string + expr: hr + type: string + File Output Operator + directory: /tmp/hive-athusoo/342842479/619377325.10000.insclause-0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + properties: + name dest1 + serialization.ddl struct dest1 { i32 key, string value, string dt, string hr} + serialization.format 1 + columns key,value,dt,hr + bucket_count -1 + serialization.lib org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe + file.inputformat org.apache.hadoop.mapred.TextInputFormat + file.outputformat org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + location file:/data/users/athusoo/hadoop_local_ws1/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/dest1 + serde: org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe + name: dest1 Needs Tagging: Path -> Alias: - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/srcpart/ds=2008-04-08/hr=11 + file:/data/users/athusoo/hadoop_local_ws1/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/srcpart/ds=2008-04-08/hr=11 Path -> Partition: - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/srcpart/ds=2008-04-08/hr=11 + file:/data/users/athusoo/hadoop_local_ws1/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/srcpart/ds=2008-04-08/hr=11 Partition partition values: ds 2008-04-08 @@ -64,7 +67,7 @@ serialization.lib org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat - location file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/srcpart + location file:/data/users/athusoo/hadoop_local_ws1/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/srcpart serde: org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe name: srcpart @@ -72,7 +75,7 @@ Move Operator tables: replace: - source: /tmp/hive-njain/1799446246/1449752317.10000.insclause-0 + source: /tmp/hive-athusoo/342842479/619377325.10000.insclause-0 table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat @@ -81,12 +84,11 @@ serialization.ddl struct dest1 { i32 key, string value, string dt, string hr} serialization.format 1 columns key,value,dt,hr - SORTBUCKETCOLSPREFIX TRUE bucket_count -1 serialization.lib org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat - location file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/dest1 + location file:/data/users/athusoo/hadoop_local_ws1/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/dest1 serde: org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe name: dest1 Index: src/contrib/hive/ql/src/test/results/clientpositive/inputddl2.q.out =================================================================== --- src/contrib/hive/ql/src/test/results/clientpositive/inputddl2.q.out (revision 711472) +++ src/contrib/hive/ql/src/test/results/clientpositive/inputddl2.q.out (working copy) @@ -1,5 +1,5 @@ ABSTRACT SYNTAX TREE: - (TOK_CREATETABLE INPUTDDL2 (TOK_TABCOLLIST (TOK_TABCOL key TOK_INT) (TOK_TABCOL value TOK_STRING)) (TOK_TABLEPARTCOLS (TOK_TABCOLLIST (TOK_TABCOL ds TOK_DATETIME) (TOK_TABCOL country TOK_STRING)))) + (TOK_CREATETABLE INPUTDDL2 (TOK_TABCOLLIST (TOK_TABCOL key TOK_INT) (TOK_TABCOL value TOK_STRING)) (TOK_TABLEPARTCOLS (TOK_TABCOLLIST (TOK_TABCOL ds TOK_DATETIME) (TOK_TABCOL country TOK_STRING))) TOK_TBLTEXTFILE) STAGE DEPENDENCIES: Stage-0 is a root stage Index: src/contrib/hive/ql/src/test/results/clientpositive/groupby1_map.q.out =================================================================== --- src/contrib/hive/ql/src/test/results/clientpositive/groupby1_map.q.out (revision 0) +++ src/contrib/hive/ql/src/test/results/clientpositive/groupby1_map.q.out (revision 0) @@ -0,0 +1,397 @@ +ABSTRACT SYNTAX TREE: + (TOK_QUERY (TOK_FROM (TOK_TABREF src)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB dest1)) (TOK_SELECT (TOK_SELEXPR (TOK_COLREF src key)) (TOK_SELEXPR (TOK_FUNCTION sum (TOK_FUNCTION substr (TOK_COLREF src value) 4)))) (TOK_GROUPBY (TOK_COLREF src key)))) + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Alias -> Map Operator Tree: + src + Group By Operator + + expr: sum(UDFToDouble(substr(value, 4))) + keys: + expr: key + type: string + mode: hash + Reduce Output Operator + key expressions: + expr: 0 + type: string + # partition fields: -1 + tag: -1 + value expressions: + expr: 1 + type: double + Reduce Operator Tree: + Group By Operator + + expr: sum(VALUE.0) + keys: + expr: KEY.0 + type: string + mode: partial2 + File Output Operator + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.mapred.SequenceFileOutputFormat + name: binary_table + + Stage: Stage-2 + Map Reduce + Alias -> Map Operator Tree: + /tmp/hive-athusoo/149566300/117302201.10001 + Reduce Output Operator + key expressions: + expr: 0 + type: string + # partition fields: 1 + tag: -1 + value expressions: + expr: 1 + type: double + Reduce Operator Tree: + Group By Operator + + expr: sum(VALUE.0) + keys: + expr: KEY.0 + type: string + mode: unknown + Select Operator + expressions: + expr: 0 + type: string + expr: 1 + type: double + File Output Operator + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe + name: dest1 + + Stage: Stage-0 + Move Operator + tables: + replace: + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe + name: dest1 + + +0 0.0 +10 10.0 +100 200.0 +103 206.0 +104 208.0 +105 105.0 +11 11.0 +111 111.0 +113 226.0 +114 114.0 +116 116.0 +118 236.0 +119 357.0 +12 24.0 +120 240.0 +125 250.0 +126 126.0 +128 384.0 +129 258.0 +131 131.0 +133 133.0 +134 268.0 +136 136.0 +137 274.0 +138 552.0 +143 143.0 +145 145.0 +146 292.0 +149 298.0 +15 30.0 +150 150.0 +152 304.0 +153 153.0 +155 155.0 +156 156.0 +157 157.0 +158 158.0 +160 160.0 +162 162.0 +163 163.0 +164 328.0 +165 330.0 +166 166.0 +167 501.0 +168 168.0 +169 676.0 +17 17.0 +170 170.0 +172 344.0 +174 348.0 +175 350.0 +176 352.0 +177 177.0 +178 178.0 +179 358.0 +18 36.0 +180 180.0 +181 181.0 +183 183.0 +186 186.0 +187 561.0 +189 189.0 +19 19.0 +190 190.0 +191 382.0 +192 192.0 +193 579.0 +194 194.0 +195 390.0 +196 196.0 +197 394.0 +199 597.0 +2 2.0 +20 20.0 +200 400.0 +201 201.0 +202 202.0 +203 406.0 +205 410.0 +207 414.0 +208 624.0 +209 418.0 +213 426.0 +214 214.0 +216 432.0 +217 434.0 +218 218.0 +219 438.0 +221 442.0 +222 222.0 +223 446.0 +224 448.0 +226 226.0 +228 228.0 +229 458.0 +230 1150.0 +233 466.0 +235 235.0 +237 474.0 +238 476.0 +239 478.0 +24 48.0 +241 241.0 +242 484.0 +244 244.0 +247 247.0 +248 248.0 +249 249.0 +252 252.0 +255 510.0 +256 512.0 +257 257.0 +258 258.0 +26 52.0 +260 260.0 +262 262.0 +263 263.0 +265 530.0 +266 266.0 +27 27.0 +272 544.0 +273 819.0 +274 274.0 +275 275.0 +277 1108.0 +278 556.0 +28 28.0 +280 560.0 +281 562.0 +282 564.0 +283 283.0 +284 284.0 +285 285.0 +286 286.0 +287 287.0 +288 576.0 +289 289.0 +291 291.0 +292 292.0 +296 296.0 +298 894.0 +30 30.0 +302 302.0 +305 305.0 +306 306.0 +307 614.0 +308 308.0 +309 618.0 +310 310.0 +311 933.0 +315 315.0 +316 948.0 +317 634.0 +318 954.0 +321 642.0 +322 644.0 +323 323.0 +325 650.0 +327 981.0 +33 33.0 +331 662.0 +332 332.0 +333 666.0 +335 335.0 +336 336.0 +338 338.0 +339 339.0 +34 34.0 +341 341.0 +342 684.0 +344 688.0 +345 345.0 +348 1740.0 +35 105.0 +351 351.0 +353 706.0 +356 356.0 +360 360.0 +362 362.0 +364 364.0 +365 365.0 +366 366.0 +367 734.0 +368 368.0 +369 1107.0 +37 74.0 +373 373.0 +374 374.0 +375 375.0 +377 377.0 +378 378.0 +379 379.0 +382 764.0 +384 1152.0 +386 386.0 +389 389.0 +392 392.0 +393 393.0 +394 394.0 +395 790.0 +396 1188.0 +397 794.0 +399 798.0 +4 4.0 +400 400.0 +401 2005.0 +402 402.0 +403 1209.0 +404 808.0 +406 1624.0 +407 407.0 +409 1227.0 +41 41.0 +411 411.0 +413 826.0 +414 828.0 +417 1251.0 +418 418.0 +419 419.0 +42 84.0 +421 421.0 +424 848.0 +427 427.0 +429 858.0 +43 43.0 +430 1290.0 +431 1293.0 +432 432.0 +435 435.0 +436 436.0 +437 437.0 +438 1314.0 +439 878.0 +44 44.0 +443 443.0 +444 444.0 +446 446.0 +448 448.0 +449 449.0 +452 452.0 +453 453.0 +454 1362.0 +455 455.0 +457 457.0 +458 916.0 +459 918.0 +460 460.0 +462 924.0 +463 926.0 +466 1398.0 +467 467.0 +468 1872.0 +469 2345.0 +47 47.0 +470 470.0 +472 472.0 +475 475.0 +477 477.0 +478 956.0 +479 479.0 +480 1440.0 +481 481.0 +482 482.0 +483 483.0 +484 484.0 +485 485.0 +487 487.0 +489 1956.0 +490 490.0 +491 491.0 +492 984.0 +493 493.0 +494 494.0 +495 495.0 +496 496.0 +497 497.0 +498 1494.0 +5 15.0 +51 102.0 +53 53.0 +54 54.0 +57 57.0 +58 116.0 +64 64.0 +65 65.0 +66 66.0 +67 134.0 +69 69.0 +70 210.0 +72 144.0 +74 74.0 +76 152.0 +77 77.0 +78 78.0 +8 8.0 +80 80.0 +82 82.0 +83 166.0 +84 168.0 +85 85.0 +86 86.0 +87 87.0 +9 9.0 +90 270.0 +92 92.0 +95 190.0 +96 96.0 +97 194.0 +98 196.0 Index: src/contrib/hive/ql/src/test/results/clientpositive/sample5.q.out =================================================================== --- src/contrib/hive/ql/src/test/results/clientpositive/sample5.q.out (revision 711472) +++ src/contrib/hive/ql/src/test/results/clientpositive/sample5.q.out (working copy) @@ -1,5 +1,5 @@ ABSTRACT SYNTAX TREE: - (TOK_QUERY (TOK_FROM (TOK_TABREF srcbucket (TOK_TABLESAMPLE 1 5 key) s)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB dest1)) (TOK_SELECT (TOK_SELEXPR (TOK_ALLCOLREF s))))) + (TOK_QUERY (TOK_FROM (TOK_TABREF srcbucket (TOK_TABLESAMPLE 1 5 (TOK_COLREF key)) s)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB dest1)) (TOK_SELECT (TOK_SELEXPR (TOK_ALLCOLREF s))))) STAGE DEPENDENCIES: Stage-1 is a root stage @@ -13,7 +13,7 @@ Filter Operator predicate: expr: (((default_sample_hashfn(key) & 2147483647) % 5) = 0) - type: Boolean + type: boolean Select Operator expressions: expr: key @@ -21,7 +21,7 @@ expr: value type: string File Output Operator - directory: /tmp/hive-njain/535229973/587370170.10000.insclause-0 + directory: /tmp/hive-athusoo/109153285/599792564.10000.insclause-0 table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat @@ -30,33 +30,33 @@ serialization.ddl struct dest1 { i32 key, string value} serialization.format 1 columns key,value - SORTBUCKETCOLSPREFIX TRUE bucket_count -1 serialization.lib org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat - location file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/dest1 + location file:/data/users/athusoo/hadoop_local_ws1/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/dest1 serde: org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe name: dest1 Needs Tagging: Path -> Alias: - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/srcbucket + file:/data/users/athusoo/hadoop_local_ws1/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/srcbucket Path -> Partition: - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/srcbucket + file:/data/users/athusoo/hadoop_local_ws1/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/srcbucket Partition input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat properties: name srcbucket + bucket_field_name key serialization.ddl struct srcbucket { string key, string value} - serialization.format 1 columns key,value + serialization.format 1 bucket_count 2 serialization.lib org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat - location file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/srcbucket + location file:/data/users/athusoo/hadoop_local_ws1/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/srcbucket serde: org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe name: srcbucket @@ -64,7 +64,7 @@ Move Operator tables: replace: - source: /tmp/hive-njain/535229973/587370170.10000.insclause-0 + source: /tmp/hive-athusoo/109153285/599792564.10000.insclause-0 table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat @@ -73,12 +73,11 @@ serialization.ddl struct dest1 { i32 key, string value} serialization.format 1 columns key,value - SORTBUCKETCOLSPREFIX TRUE bucket_count -1 serialization.lib org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat - location file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/dest1 + location file:/data/users/athusoo/hadoop_local_ws1/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/dest1 serde: org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe name: dest1 Index: src/contrib/hive/ql/src/test/results/clientpositive/groupby1_limit.q.out =================================================================== --- src/contrib/hive/ql/src/test/results/clientpositive/groupby1_limit.q.out (revision 711472) +++ src/contrib/hive/ql/src/test/results/clientpositive/groupby1_limit.q.out (working copy) @@ -23,20 +23,21 @@ Reduce Operator Tree: Group By Operator - expr: sum(VALUE.0) + expr: sum(UDFToDouble(VALUE.0)) keys: expr: KEY.0 type: string mode: partial1 File Output Operator table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.mapred.SequenceFileOutputFormat + name: binary_table Stage: Stage-2 Map Reduce Alias -> Map Operator Tree: - /tmp/hive-njain/7427260/341902671.10001 + /tmp/hive-athusoo/614159520/87445592.10001 Reduce Output Operator key expressions: expr: 0 @@ -45,7 +46,7 @@ tag: -1 value expressions: expr: 1 - type: string + type: double Reduce Operator Tree: Group By Operator @@ -53,13 +54,13 @@ keys: expr: KEY.0 type: string - mode: partial2 + mode: unknown Select Operator expressions: expr: 0 type: string expr: 1 - type: string + type: double Limit File Output Operator table: Index: src/contrib/hive/ql/src/test/results/clientpositive/inputddl4.q.out =================================================================== --- src/contrib/hive/ql/src/test/results/clientpositive/inputddl4.q.out (revision 711472) +++ src/contrib/hive/ql/src/test/results/clientpositive/inputddl4.q.out (working copy) @@ -17,4 +17,4 @@ ds datetime country string Detailed Table Information: -Table(tableName:inputddl4,dbName:default,owner:njain,createTime:1224285030,lastAccessTime:0,retention:0,sd:StorageDescriptor(cols:[FieldSchema(name:viewtime,type:datetime,comment:null), FieldSchema(name:userid,type:int,comment:null), FieldSchema(name:page_url,type:string,comment:null), FieldSchema(name:referrer_url,type:string,comment:null), FieldSchema(name:friends,type:array,comment:null), FieldSchema(name:properties,type:map,comment:null), FieldSchema(name:ip,type:string,comment:'IP Address of the User')],location:file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/inputddl4,inputFormat:org.apache.hadoop.mapred.TextInputFormat,outputFormat:org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat,compressed:false,numBuckets:32,serdeInfo:SerDeInfo(name:null,serializationLib:org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe,parameters:{serialization.format=1}),bucketCols:[userid],sortCols:[Order(col:viewtime,order:1)],parameters:{}),partitionKeys:[FieldSchema(name:ds,type:datetime,comment:null), FieldSchema(name:country,type:string,comment:null)],parameters:{comment='This is the page view table'}) +Table(tableName:inputddl4,dbName:default,owner:athusoo,createTime:1225763924,lastAccessTime:0,retention:0,sd:StorageDescriptor(cols:[FieldSchema(name:viewtime,type:datetime,comment:null), FieldSchema(name:userid,type:int,comment:null), FieldSchema(name:page_url,type:string,comment:null), FieldSchema(name:referrer_url,type:string,comment:null), FieldSchema(name:friends,type:array,comment:null), FieldSchema(name:properties,type:map,comment:null), FieldSchema(name:ip,type:string,comment:IP Address of the User)],location:file:/data/users/athusoo/hadoop_local_ws1/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/inputddl4,inputFormat:org.apache.hadoop.mapred.TextInputFormat,outputFormat:org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat,compressed:false,numBuckets:32,serdeInfo:SerDeInfo(name:null,serializationLib:org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe,parameters:{serialization.format=1}),bucketCols:[userid],sortCols:[Order(col:viewtime,order:1)],parameters:{}),partitionKeys:[FieldSchema(name:ds,type:datetime,comment:null), FieldSchema(name:country,type:string,comment:null)],parameters:{comment=This is the page view table}) Index: src/contrib/hive/ql/src/test/results/clientpositive/groupby2_limit.q.out =================================================================== --- src/contrib/hive/ql/src/test/results/clientpositive/groupby2_limit.q.out (revision 711472) +++ src/contrib/hive/ql/src/test/results/clientpositive/groupby2_limit.q.out (working copy) @@ -23,20 +23,21 @@ Reduce Operator Tree: Group By Operator - expr: sum(VALUE.0) + expr: sum(UDFToDouble(VALUE.0)) keys: expr: KEY.0 type: string mode: partial1 File Output Operator table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.mapred.SequenceFileOutputFormat + name: binary_table Stage: Stage-2 Map Reduce Alias -> Map Operator Tree: - /tmp/hive-njain/606295988/175965730.10002 + /tmp/hive-athusoo/835000263/890252209.10002 Reduce Output Operator key expressions: expr: 0 @@ -45,7 +46,7 @@ tag: -1 value expressions: expr: 1 - type: string + type: double Reduce Operator Tree: Group By Operator @@ -53,13 +54,13 @@ keys: expr: KEY.0 type: string - mode: partial2 + mode: unknown Select Operator expressions: expr: 0 type: string expr: 1 - type: string + type: double Limit File Output Operator table: Index: src/contrib/hive/ql/src/test/results/clientpositive/groupby3_map.q.out =================================================================== --- src/contrib/hive/ql/src/test/results/clientpositive/groupby3_map.q.out (revision 0) +++ src/contrib/hive/ql/src/test/results/clientpositive/groupby3_map.q.out (revision 0) @@ -0,0 +1,114 @@ +ABSTRACT SYNTAX TREE: + (TOK_QUERY (TOK_FROM (TOK_TABREF src)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB dest1)) (TOK_SELECT (TOK_SELEXPR (TOK_FUNCTION sum (TOK_FUNCTION substr (TOK_COLREF src value) 4))) (TOK_SELEXPR (TOK_FUNCTION avg (TOK_FUNCTION substr (TOK_COLREF src value) 4))) (TOK_SELEXPR (TOK_FUNCTIONDI avg (TOK_FUNCTION substr (TOK_COLREF src value) 4))) (TOK_SELEXPR (TOK_FUNCTION max (TOK_FUNCTION substr (TOK_COLREF src value) 4))) (TOK_SELEXPR (TOK_FUNCTION min (TOK_FUNCTION substr (TOK_COLREF src value) 4)))))) + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Alias -> Map Operator Tree: + src + Group By Operator + + expr: avg(DISTINCT UDFToDouble(substr(value, 4))) + expr: sum(UDFToDouble(substr(value, 4))) + expr: avg(UDFToDouble(substr(value, 4))) + expr: min(UDFToDouble(substr(value, 4))) + expr: max(UDFToDouble(substr(value, 4))) + keys: + expr: substr(value, 4) + type: string + mode: hash + Reduce Output Operator + key expressions: + expr: 0 + type: string + # partition fields: 2147483647 + tag: -1 + value expressions: + expr: 1 + type: string + expr: 2 + type: double + expr: 3 + type: string + expr: 4 + type: double + expr: 5 + type: double + Reduce Operator Tree: + Group By Operator + + expr: avg(DISTINCT UDFToDouble(KEY.0)) + expr: sum(VALUE.1) + expr: avg(VALUE.2) + expr: min(VALUE.3) + expr: max(VALUE.4) + mode: partial2 + File Output Operator + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.mapred.SequenceFileOutputFormat + name: binary_table + + Stage: Stage-2 + Map Reduce + Alias -> Map Operator Tree: + /tmp/hive-athusoo/118688352/1328529959.10001 + Reduce Output Operator + # partition fields: 0 + tag: -1 + value expressions: + expr: 0 + type: string + expr: 1 + type: double + expr: 2 + type: string + expr: 3 + type: double + expr: 4 + type: double + Reduce Operator Tree: + Group By Operator + + expr: avg(VALUE.0) + expr: sum(VALUE.1) + expr: avg(VALUE.2) + expr: min(VALUE.3) + expr: max(VALUE.4) + mode: unknown + Select Operator + expressions: + expr: 1 + type: double + expr: 2 + type: string + expr: 0 + type: string + expr: 4 + type: double + expr: 3 + type: double + File Output Operator + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe + name: dest1 + + Stage: Stage-0 + Move Operator + tables: + replace: + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe + name: dest1 + + +130091.0 260.182 256.10355987055016 498.0 0.0 Index: src/contrib/hive/ql/src/test/results/clientpositive/sample7.q.out =================================================================== --- src/contrib/hive/ql/src/test/results/clientpositive/sample7.q.out (revision 711472) +++ src/contrib/hive/ql/src/test/results/clientpositive/sample7.q.out (working copy) @@ -1,5 +1,5 @@ ABSTRACT SYNTAX TREE: - (TOK_QUERY (TOK_FROM (TOK_TABREF srcbucket (TOK_TABLESAMPLE 1 4 key) s)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB dest1)) (TOK_SELECT (TOK_SELEXPR (TOK_ALLCOLREF s))) (TOK_WHERE (> (TOK_COLREF s key) 100)))) + (TOK_QUERY (TOK_FROM (TOK_TABREF srcbucket (TOK_TABLESAMPLE 1 4 (TOK_COLREF key)) s)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB dest1)) (TOK_SELECT (TOK_SELEXPR (TOK_ALLCOLREF s))) (TOK_WHERE (> (TOK_COLREF s key) 100)))) STAGE DEPENDENCIES: Stage-1 is a root stage @@ -13,11 +13,11 @@ Filter Operator predicate: expr: (((default_sample_hashfn(key) & 2147483647) % 4) = 0) - type: Boolean + type: boolean Filter Operator predicate: expr: (key > 100) - type: Boolean + type: boolean Select Operator expressions: expr: key @@ -25,7 +25,7 @@ expr: value type: string File Output Operator - directory: /tmp/hive-njain/1265167991/144531187.10000.insclause-0 + directory: /tmp/hive-athusoo/129140824/559107802.10000.insclause-0 table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat @@ -34,33 +34,33 @@ serialization.ddl struct dest1 { i32 key, string value} serialization.format 1 columns key,value - SORTBUCKETCOLSPREFIX TRUE bucket_count -1 serialization.lib org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat - location file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/dest1 + location file:/data/users/athusoo/hadoop_local_ws1/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/dest1 serde: org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe name: dest1 Needs Tagging: Path -> Alias: - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/srcbucket + file:/data/users/athusoo/hadoop_local_ws1/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/srcbucket/kv1.txt Path -> Partition: - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/srcbucket + file:/data/users/athusoo/hadoop_local_ws1/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/srcbucket/kv1.txt Partition input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat properties: name srcbucket + bucket_field_name key serialization.ddl struct srcbucket { string key, string value} - serialization.format 1 columns key,value + serialization.format 1 bucket_count 2 serialization.lib org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat - location file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/srcbucket + location file:/data/users/athusoo/hadoop_local_ws1/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/srcbucket serde: org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe name: srcbucket @@ -68,7 +68,7 @@ Move Operator tables: replace: - source: /tmp/hive-njain/1265167991/144531187.10000.insclause-0 + source: /tmp/hive-athusoo/129140824/559107802.10000.insclause-0 table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat @@ -77,12 +77,11 @@ serialization.ddl struct dest1 { i32 key, string value} serialization.format 1 columns key,value - SORTBUCKETCOLSPREFIX TRUE bucket_count -1 serialization.lib org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat - location file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/dest1 + location file:/data/users/athusoo/hadoop_local_ws1/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/dest1 serde: org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe name: dest1 @@ -181,113 +180,3 @@ 477 val_477 169 val_169 400 val_400 -271 val_272 -217 val_218 -455 val_456 -231 val_232 -448 val_449 -246 val_247 -440 val_441 -147 val_148 -356 val_357 -217 val_218 -110 val_111 -275 val_276 -293 val_294 -286 val_287 -408 val_409 -477 val_478 -455 val_456 -367 val_368 -488 val_489 -349 val_350 -161 val_162 -224 val_225 -206 val_207 -114 val_115 -239 val_240 -389 val_390 -114 val_115 -235 val_236 -165 val_166 -264 val_265 -129 val_130 -257 val_258 -323 val_324 -411 val_412 -338 val_339 -147 val_148 -473 val_474 -213 val_214 -426 val_427 -132 val_133 -121 val_122 -129 val_130 -246 val_247 -491 val_492 -352 val_353 -349 val_350 -480 val_481 -480 val_481 -183 val_184 -392 val_393 -275 val_276 -114 val_115 -161 val_162 -264 val_265 -484 val_485 -305 val_306 -367 val_368 -260 val_261 -349 val_350 -363 val_364 -118 val_119 -121 val_122 -239 val_240 -143 val_144 -341 val_342 -363 val_364 -293 val_294 -206 val_207 -381 val_382 -404 val_405 -206 val_207 -385 val_386 -260 val_261 -404 val_405 -451 val_452 -132 val_133 -356 val_357 -352 val_353 -330 val_331 -257 val_258 -374 val_375 -437 val_438 -334 val_335 -404 val_405 -268 val_269 -491 val_492 -392 val_393 -341 val_342 -118 val_119 -172 val_173 -136 val_137 -349 val_350 -462 val_463 -408 val_409 -378 val_379 -495 val_496 -385 val_386 -389 val_390 -125 val_126 -415 val_416 -228 val_229 -363 val_364 -459 val_460 -341 val_342 -440 val_441 -309 val_310 -161 val_162 -161 val_162 -118 val_119 -242 val_243 Index: src/contrib/hive/ql/src/test/results/clientpositive/inputddl6.q.out =================================================================== --- src/contrib/hive/ql/src/test/results/clientpositive/inputddl6.q.out (revision 711472) +++ src/contrib/hive/ql/src/test/results/clientpositive/inputddl6.q.out (working copy) @@ -2,13 +2,13 @@ value string ds datetime Detailed Table Information: -Table(tableName:inputddl6,dbName:default,owner:njain,createTime:1224285279,lastAccessTime:0,retention:0,sd:StorageDescriptor(cols:[FieldSchema(name:key,type:string,comment:null), FieldSchema(name:value,type:string,comment:null)],location:file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/inputddl6,inputFormat:org.apache.hadoop.mapred.TextInputFormat,outputFormat:org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat,compressed:false,numBuckets:-1,serdeInfo:SerDeInfo(name:null,serializationLib:org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe,parameters:{serialization.format=1}),bucketCols:[],sortCols:[],parameters:{}),partitionKeys:[FieldSchema(name:ds,type:datetime,comment:null)],parameters:{SORTBUCKETCOLSPREFIX=TRUE}) +Table(tableName:inputddl6,dbName:default,owner:athusoo,createTime:1225763929,lastAccessTime:0,retention:0,sd:StorageDescriptor(cols:[FieldSchema(name:key,type:string,comment:null), FieldSchema(name:value,type:string,comment:null)],location:file:/data/users/athusoo/hadoop_local_ws1/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/inputddl6,inputFormat:org.apache.hadoop.mapred.TextInputFormat,outputFormat:org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat,compressed:false,numBuckets:-1,serdeInfo:SerDeInfo(name:null,serializationLib:org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe,parameters:{serialization.format=1}),bucketCols:[],sortCols:[],parameters:{}),partitionKeys:[FieldSchema(name:ds,type:datetime,comment:null)],parameters:{}) key string value string ds datetime Detailed Partition Information: -Partition(values:[2008-04-08],dbName:default,tableName:inputddl6,createTime:0,lastAccessTime:0,sd:StorageDescriptor(cols:[FieldSchema(name:key,type:string,comment:null), FieldSchema(name:value,type:string,comment:null)],location:file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/inputddl6/ds=2008-04-08,inputFormat:org.apache.hadoop.mapred.TextInputFormat,outputFormat:org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat,compressed:false,numBuckets:-1,serdeInfo:SerDeInfo(name:null,serializationLib:org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe,parameters:{serialization.format=1}),bucketCols:[],sortCols:[],parameters:{}),parameters:{}) +Partition(values:[2008-04-08],dbName:default,tableName:inputddl6,createTime:0,lastAccessTime:0,sd:StorageDescriptor(cols:[FieldSchema(name:key,type:string,comment:null), FieldSchema(name:value,type:string,comment:null)],location:file:/data/users/athusoo/hadoop_local_ws1/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/inputddl6/ds=2008-04-08,inputFormat:org.apache.hadoop.mapred.TextInputFormat,outputFormat:org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat,compressed:false,numBuckets:-1,serdeInfo:SerDeInfo(name:null,serializationLib:org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe,parameters:{serialization.format=1}),bucketCols:[],sortCols:[],parameters:{}),parameters:{}) ds=2008-04-08 ds=2008-04-09 ds=2008-04-09 Index: src/contrib/hive/ql/src/test/results/clientpositive/groupby5_map.q.out =================================================================== --- src/contrib/hive/ql/src/test/results/clientpositive/groupby5_map.q.out (revision 0) +++ src/contrib/hive/ql/src/test/results/clientpositive/groupby5_map.q.out (revision 0) @@ -0,0 +1,50 @@ +ABSTRACT SYNTAX TREE: + (TOK_QUERY (TOK_FROM (TOK_TABREF src)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB dest1)) (TOK_SELECT (TOK_SELEXPR (TOK_FUNCTION sum (TOK_COLREF src key)))))) + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Alias -> Map Operator Tree: + src + Group By Operator + + expr: sum(UDFToDouble(key)) + mode: hash + Reduce Output Operator + # partition fields: -1 + tag: -1 + value expressions: + expr: 0 + type: double + Reduce Operator Tree: + Group By Operator + + expr: sum(VALUE.0) + mode: unknown + Select Operator + expressions: + expr: 0 + type: double + File Output Operator + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe + name: dest1 + + Stage: Stage-0 + Move Operator + tables: + replace: + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe + name: dest1 + + +130091.0 Index: src/contrib/hive/ql/src/test/results/compiler/parse/cast1.q.out =================================================================== --- src/contrib/hive/ql/src/test/results/compiler/parse/cast1.q.out (revision 711472) +++ src/contrib/hive/ql/src/test/results/compiler/parse/cast1.q.out (working copy) @@ -1 +1 @@ -(TOK_QUERY (TOK_FROM (TOK_TABREF src)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB dest1)) (TOK_SELECT (TOK_SELEXPR (+ 3 2)) (TOK_SELEXPR (+ 3.0 2)) (TOK_SELEXPR (+ 3 2.0)) (TOK_SELEXPR (+ 3.0 2.0)) (TOK_SELEXPR (+ 3 (TOK_FUNCTION TOK_INT 2.0))) (TOK_SELEXPR (TOK_FUNCTION TOK_BOOLEAN 1)) (TOK_SELEXPR (TOK_FUNCTION TOK_INT TRUE))) (TOK_WHERE (= (TOK_COLREF src key) 86)))) null \ No newline at end of file +(TOK_QUERY (TOK_FROM (TOK_TABREF src)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (+ 3 2)) (TOK_SELEXPR (+ 3.0 2)) (TOK_SELEXPR (+ 3 2.0)) (TOK_SELEXPR (+ 3.0 2.0)) (TOK_SELEXPR (+ 3 (TOK_FUNCTION TOK_INT 2.0))) (TOK_SELEXPR (TOK_FUNCTION TOK_BOOLEAN 1)) (TOK_SELEXPR (TOK_FUNCTION TOK_INT TRUE))) (TOK_WHERE (= (TOK_COLREF src key) 86)))) null \ No newline at end of file Index: src/contrib/hive/ql/src/test/results/compiler/parse/input2.q.out =================================================================== --- src/contrib/hive/ql/src/test/results/compiler/parse/input2.q.out (revision 711472) +++ src/contrib/hive/ql/src/test/results/compiler/parse/input2.q.out (working copy) @@ -1 +1 @@ -(TOK_QUERY (TOK_FROM (TOK_TABREF src)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB dest1)) (TOK_SELECT (TOK_SELEXPR (TOK_ALLCOLREF src))) (TOK_WHERE (< (TOK_COLREF src key) 100))) (TOK_INSERT (TOK_DESTINATION (TOK_TAB dest2)) (TOK_SELECT (TOK_SELEXPR (TOK_COLREF src key)) (TOK_SELEXPR (TOK_COLREF src value))) (TOK_WHERE (and (>= (TOK_COLREF src key) 100) (< (TOK_COLREF src key) 200)))) (TOK_INSERT (TOK_DESTINATION (TOK_TAB dest3 (TOK_PARTSPEC (TOK_PARTVAL ds '2008-04-08') (TOK_PARTVAL hr '12')))) (TOK_SELECT (TOK_SELEXPR (TOK_COLREF src key))) (TOK_WHERE (>= (TOK_COLREF src key) 200)))) null \ No newline at end of file +(TOK_QUERY (TOK_FROM (TOK_TABREF src)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB dest1)) (TOK_SELECT (TOK_SELEXPR (TOK_ALLCOLREF src))) (TOK_WHERE (< (TOK_COLREF src key) 100))) (TOK_INSERT (TOK_DESTINATION (TOK_TAB dest2)) (TOK_SELECT (TOK_SELEXPR (TOK_COLREF src key)) (TOK_SELEXPR (TOK_COLREF src value))) (TOK_WHERE (and (>= (TOK_COLREF src key) 100) (< (TOK_COLREF src key) 200)))) (TOK_INSERT (TOK_DESTINATION (TOK_TAB dest3 (TOK_PARTSPEC (TOK_PARTVAL ds '2008-04-08') (TOK_PARTVAL hr '12')))) (TOK_SELECT (TOK_SELEXPR (TOK_COLREF src key)) (TOK_SELEXPR 2)) (TOK_WHERE (>= (TOK_COLREF src key) 200)))) null \ No newline at end of file Index: src/contrib/hive/ql/src/test/results/compiler/parse/input3.q.out =================================================================== --- src/contrib/hive/ql/src/test/results/compiler/parse/input3.q.out (revision 711472) +++ src/contrib/hive/ql/src/test/results/compiler/parse/input3.q.out (working copy) @@ -1 +1 @@ -(TOK_QUERY (TOK_FROM (TOK_TABREF src)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB dest1)) (TOK_SELECT (TOK_SELEXPR (TOK_ALLCOLREF src))) (TOK_WHERE (< (TOK_COLREF src key) 100))) (TOK_INSERT (TOK_DESTINATION (TOK_TAB dest2)) (TOK_SELECT (TOK_SELEXPR (TOK_COLREF src key)) (TOK_SELEXPR (TOK_COLREF src value))) (TOK_WHERE (and (>= (TOK_COLREF src key) 100) (< (TOK_COLREF src key) 200)))) (TOK_INSERT (TOK_DESTINATION (TOK_TAB dest3 (TOK_PARTSPEC (TOK_PARTVAL ds '2008-04-08') (TOK_PARTVAL hr '12')))) (TOK_SELECT (TOK_SELEXPR (TOK_COLREF src key))) (TOK_WHERE (and (>= (TOK_COLREF src key) 200) (< (TOK_COLREF src key) 300)))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR '../../../../build/contrib/hive/ql/test/data/warehouse/dest4.out')) (TOK_SELECT (TOK_SELEXPR (TOK_COLREF src value))) (TOK_WHERE (>= (TOK_COLREF src key) 300)))) null \ No newline at end of file +(TOK_QUERY (TOK_FROM (TOK_TABREF src)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB dest1)) (TOK_SELECT (TOK_SELEXPR (TOK_ALLCOLREF src))) (TOK_WHERE (< (TOK_COLREF src key) 100))) (TOK_INSERT (TOK_DESTINATION (TOK_TAB dest2)) (TOK_SELECT (TOK_SELEXPR (TOK_COLREF src key)) (TOK_SELEXPR (TOK_COLREF src value))) (TOK_WHERE (and (>= (TOK_COLREF src key) 100) (< (TOK_COLREF src key) 200)))) (TOK_INSERT (TOK_DESTINATION (TOK_TAB dest3 (TOK_PARTSPEC (TOK_PARTVAL ds '2008-04-08') (TOK_PARTVAL hr '12')))) (TOK_SELECT (TOK_SELEXPR (TOK_COLREF src key)) (TOK_SELEXPR 2)) (TOK_WHERE (and (>= (TOK_COLREF src key) 200) (< (TOK_COLREF src key) 300)))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR '../../../../build/contrib/hive/ql/test/data/warehouse/dest4.out')) (TOK_SELECT (TOK_SELEXPR (TOK_COLREF src value))) (TOK_WHERE (>= (TOK_COLREF src key) 300)))) null \ No newline at end of file Index: src/contrib/hive/ql/src/test/results/compiler/parse/join4.q.out =================================================================== --- src/contrib/hive/ql/src/test/results/compiler/parse/join4.q.out (revision 711472) +++ src/contrib/hive/ql/src/test/results/compiler/parse/join4.q.out (working copy) @@ -1 +1 @@ -(TOK_QUERY (TOK_FROM (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_LEFTOUTERJOIN (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF src src1)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_COLREF src1 key) c1) (TOK_SELEXPR (TOK_COLREF src1 value) c2)) (TOK_WHERE (and (> (TOK_COLREF src1 key) 10) (< (TOK_COLREF src1 key) 20))))) a) (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF src src2)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_COLREF src2 key) c3) (TOK_SELEXPR (TOK_COLREF src2 value) c4)) (TOK_WHERE (and (> (TOK_COLREF src2 key) 15) (< (TOK_COLREF src2 key) 25))))) b) (= (TOK_COLREF a c1) (TOK_COLREF b c3)))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_COLREF a c1) c1) (TOK_SELEXPR (TOK_COLREF a c2) c2) (TOK_SELEXPR (TOK_COLREF b c3) c3) (TOK_SELEXPR (TOK_COLREF b c4) c4)))) c)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB dest1)) (TOK_SELECT (TOK_SELEXPR (TOK_COLREF c c1)) (TOK_SELEXPR (TOK_COLREF c c2)) (TOK_SELEXPR (TOK_COLREF c c3)) (TOK_SELEXPR (TOK_COLREF c c4))))) null \ No newline at end of file +(TOK_QUERY (TOK_FROM (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_LEFTOUTERJOIN (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF src src1)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_COLREF src1 key) c1) (TOK_SELEXPR (TOK_COLREF src1 value) c2)) (TOK_WHERE (and (> (TOK_COLREF src1 key) 10) (< (TOK_COLREF src1 key) 20))))) a) (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF src src2)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_COLREF src2 key) c3) (TOK_SELEXPR (TOK_COLREF src2 value) c4)) (TOK_WHERE (and (> (TOK_COLREF src2 key) 15) (< (TOK_COLREF src2 key) 25))))) b) (= (TOK_COLREF a c1) (TOK_COLREF b c3)))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_COLREF a c1) c1) (TOK_SELEXPR (TOK_COLREF a c2) c2) (TOK_SELEXPR (TOK_COLREF b c3) c3) (TOK_SELEXPR (TOK_COLREF b c4) c4)))) c)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_COLREF c c1)) (TOK_SELEXPR (TOK_COLREF c c2)) (TOK_SELEXPR (TOK_COLREF c c3)) (TOK_SELEXPR (TOK_COLREF c c4))))) null \ No newline at end of file Index: src/contrib/hive/ql/src/test/results/compiler/parse/input4.q.out =================================================================== --- src/contrib/hive/ql/src/test/results/compiler/parse/input4.q.out (revision 711472) +++ src/contrib/hive/ql/src/test/results/compiler/parse/input4.q.out (working copy) @@ -1 +1 @@ -(TOK_QUERY (TOK_FROM (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF src)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TRANSFORM (TOK_COLLIST (TOK_COLREF src key) (TOK_COLREF src value)) (TOK_ALIASLIST tkey tvalue) '/bin/cat'))) (TOK_CLUSTERBY tkey))) tmap)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB dest1)) (TOK_SELECT (TOK_SELEXPR (TOK_COLREF tmap tkey)) (TOK_SELEXPR (TOK_COLREF tmap tvalue))) (TOK_WHERE (< (TOK_COLREF tmap tkey) 100)))) null \ No newline at end of file +(TOK_QUERY (TOK_FROM (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF src)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TRANSFORM (TOK_EXPLIST (TOK_COLREF src key) (TOK_COLREF src value)) (TOK_ALIASLIST tkey tvalue) '/bin/cat'))) (TOK_CLUSTERBY tkey))) tmap)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB dest1)) (TOK_SELECT (TOK_SELEXPR (TOK_COLREF tmap tkey)) (TOK_SELEXPR (TOK_COLREF tmap tvalue))) (TOK_WHERE (< (TOK_COLREF tmap tkey) 100)))) null \ No newline at end of file Index: src/contrib/hive/ql/src/test/results/compiler/parse/join5.q.out =================================================================== --- src/contrib/hive/ql/src/test/results/compiler/parse/join5.q.out (revision 711472) +++ src/contrib/hive/ql/src/test/results/compiler/parse/join5.q.out (working copy) @@ -1 +1 @@ -(TOK_QUERY (TOK_FROM (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_RIGHTOUTERJOIN (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF src src1)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_COLREF src1 key) c1) (TOK_SELEXPR (TOK_COLREF src1 value) c2)) (TOK_WHERE (and (> (TOK_COLREF src1 key) 10) (< (TOK_COLREF src1 key) 20))))) a) (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF src src2)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_COLREF src2 key) c3) (TOK_SELEXPR (TOK_COLREF src2 value) c4)) (TOK_WHERE (and (> (TOK_COLREF src2 key) 15) (< (TOK_COLREF src2 key) 25))))) b) (= (TOK_COLREF a c1) (TOK_COLREF b c3)))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_COLREF a c1) c1) (TOK_SELEXPR (TOK_COLREF a c2) c2) (TOK_SELEXPR (TOK_COLREF b c3) c3) (TOK_SELEXPR (TOK_COLREF b c4) c4)))) c)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB dest1)) (TOK_SELECT (TOK_SELEXPR (TOK_COLREF c c1)) (TOK_SELEXPR (TOK_COLREF c c2)) (TOK_SELEXPR (TOK_COLREF c c3)) (TOK_SELEXPR (TOK_COLREF c c4))))) null \ No newline at end of file +(TOK_QUERY (TOK_FROM (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_RIGHTOUTERJOIN (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF src src1)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_COLREF src1 key) c1) (TOK_SELEXPR (TOK_COLREF src1 value) c2)) (TOK_WHERE (and (> (TOK_COLREF src1 key) 10) (< (TOK_COLREF src1 key) 20))))) a) (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF src src2)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_COLREF src2 key) c3) (TOK_SELEXPR (TOK_COLREF src2 value) c4)) (TOK_WHERE (and (> (TOK_COLREF src2 key) 15) (< (TOK_COLREF src2 key) 25))))) b) (= (TOK_COLREF a c1) (TOK_COLREF b c3)))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_COLREF a c1) c1) (TOK_SELEXPR (TOK_COLREF a c2) c2) (TOK_SELEXPR (TOK_COLREF b c3) c3) (TOK_SELEXPR (TOK_COLREF b c4) c4)))) c)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_COLREF c c1)) (TOK_SELEXPR (TOK_COLREF c c2)) (TOK_SELEXPR (TOK_COLREF c c3)) (TOK_SELEXPR (TOK_COLREF c c4))))) null \ No newline at end of file Index: src/contrib/hive/ql/src/test/results/compiler/parse/input5.q.out =================================================================== --- src/contrib/hive/ql/src/test/results/compiler/parse/input5.q.out (revision 711472) +++ src/contrib/hive/ql/src/test/results/compiler/parse/input5.q.out (working copy) @@ -1 +1 @@ -(TOK_QUERY (TOK_FROM (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF src_thrift)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TRANSFORM (TOK_COLLIST (TOK_COLREF src_thrift lint) (TOK_COLREF src_thrift lintstring)) (TOK_ALIASLIST tkey tvalue) '/bin/cat'))) (TOK_CLUSTERBY tkey))) tmap)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB dest1)) (TOK_SELECT (TOK_SELEXPR (TOK_COLREF tmap tkey)) (TOK_SELEXPR (TOK_COLREF tmap tvalue))))) null \ No newline at end of file +(TOK_QUERY (TOK_FROM (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF src_thrift)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TRANSFORM (TOK_EXPLIST (TOK_COLREF src_thrift lint) (TOK_COLREF src_thrift lintstring)) (TOK_ALIASLIST tkey tvalue) '/bin/cat'))) (TOK_CLUSTERBY tkey))) tmap)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB dest1)) (TOK_SELECT (TOK_SELEXPR (TOK_COLREF tmap tkey)) (TOK_SELEXPR (TOK_COLREF tmap tvalue))))) null \ No newline at end of file Index: src/contrib/hive/ql/src/test/results/compiler/parse/join6.q.out =================================================================== --- src/contrib/hive/ql/src/test/results/compiler/parse/join6.q.out (revision 711472) +++ src/contrib/hive/ql/src/test/results/compiler/parse/join6.q.out (working copy) @@ -1 +1 @@ -(TOK_QUERY (TOK_FROM (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_FULLOUTERJOIN (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF src src1)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_COLREF src1 key) c1) (TOK_SELEXPR (TOK_COLREF src1 value) c2)) (TOK_WHERE (and (> (TOK_COLREF src1 key) 10) (< (TOK_COLREF src1 key) 20))))) a) (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF src src2)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_COLREF src2 key) c3) (TOK_SELEXPR (TOK_COLREF src2 value) c4)) (TOK_WHERE (and (> (TOK_COLREF src2 key) 15) (< (TOK_COLREF src2 key) 25))))) b) (= (TOK_COLREF a c1) (TOK_COLREF b c3)))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_COLREF a c1) c1) (TOK_SELEXPR (TOK_COLREF a c2) c2) (TOK_SELEXPR (TOK_COLREF b c3) c3) (TOK_SELEXPR (TOK_COLREF b c4) c4)))) c)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB dest1)) (TOK_SELECT (TOK_SELEXPR (TOK_COLREF c c1)) (TOK_SELEXPR (TOK_COLREF c c2)) (TOK_SELEXPR (TOK_COLREF c c3)) (TOK_SELEXPR (TOK_COLREF c c4))))) null \ No newline at end of file +(TOK_QUERY (TOK_FROM (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_FULLOUTERJOIN (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF src src1)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_COLREF src1 key) c1) (TOK_SELEXPR (TOK_COLREF src1 value) c2)) (TOK_WHERE (and (> (TOK_COLREF src1 key) 10) (< (TOK_COLREF src1 key) 20))))) a) (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF src src2)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_COLREF src2 key) c3) (TOK_SELEXPR (TOK_COLREF src2 value) c4)) (TOK_WHERE (and (> (TOK_COLREF src2 key) 15) (< (TOK_COLREF src2 key) 25))))) b) (= (TOK_COLREF a c1) (TOK_COLREF b c3)))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_COLREF a c1) c1) (TOK_SELEXPR (TOK_COLREF a c2) c2) (TOK_SELEXPR (TOK_COLREF b c3) c3) (TOK_SELEXPR (TOK_COLREF b c4) c4)))) c)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_COLREF c c1)) (TOK_SELEXPR (TOK_COLREF c c2)) (TOK_SELEXPR (TOK_COLREF c c3)) (TOK_SELEXPR (TOK_COLREF c c4))))) null \ No newline at end of file Index: src/contrib/hive/ql/src/test/results/compiler/parse/input_testxpath2.q.out =================================================================== --- src/contrib/hive/ql/src/test/results/compiler/parse/input_testxpath2.q.out (revision 711472) +++ src/contrib/hive/ql/src/test/results/compiler/parse/input_testxpath2.q.out (working copy) @@ -1 +1 @@ -(TOK_QUERY (TOK_FROM (TOK_TABREF src_thrift)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB dest1)) (TOK_SELECT (TOK_SELEXPR (TOK_FUNCTION size (TOK_COLREF src_thrift lint))) (TOK_SELEXPR (TOK_FUNCTION size (TOK_COLREF src_thrift lintstring))) (TOK_SELEXPR (TOK_FUNCTION size (TOK_COLREF src_thrift mstringstring)))) (TOK_WHERE (AND (TOK_FUNCTION TOK_ISNOTNULL (TOK_COLREF src_thrift lint)) (NOT (TOK_FUNCTION TOK_ISNULL (TOK_COLREF src_thrift mstringstring))))))) null \ No newline at end of file +(TOK_QUERY (TOK_FROM (TOK_TABREF src_thrift)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_FUNCTION size (TOK_COLREF src_thrift lint))) (TOK_SELEXPR (TOK_FUNCTION size (TOK_COLREF src_thrift lintstring))) (TOK_SELEXPR (TOK_FUNCTION size (TOK_COLREF src_thrift mstringstring)))) (TOK_WHERE (AND (TOK_FUNCTION TOK_ISNOTNULL (TOK_COLREF src_thrift lint)) (NOT (TOK_FUNCTION TOK_ISNULL (TOK_COLREF src_thrift mstringstring))))))) null \ No newline at end of file Index: src/contrib/hive/ql/src/test/results/compiler/parse/join7.q.out =================================================================== --- src/contrib/hive/ql/src/test/results/compiler/parse/join7.q.out (revision 711472) +++ src/contrib/hive/ql/src/test/results/compiler/parse/join7.q.out (working copy) @@ -1 +1 @@ -(TOK_QUERY (TOK_FROM (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_LEFTOUTERJOIN (TOK_FULLOUTERJOIN (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF src src1)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_COLREF src1 key) c1) (TOK_SELEXPR (TOK_COLREF src1 value) c2)) (TOK_WHERE (and (> (TOK_COLREF src1 key) 10) (< (TOK_COLREF src1 key) 20))))) a) (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF src src2)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_COLREF src2 key) c3) (TOK_SELEXPR (TOK_COLREF src2 value) c4)) (TOK_WHERE (and (> (TOK_COLREF src2 key) 15) (< (TOK_COLREF src2 key) 25))))) b) (= (TOK_COLREF a c1) (TOK_COLREF b c3))) (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF src src3)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_COLREF src3 key) c5) (TOK_SELEXPR (TOK_COLREF src3 value) c6)) (TOK_WHERE (and (> (TOK_COLREF src3 key) 20) (< (TOK_COLREF src3 key) 25))))) c) (= (TOK_COLREF a c1) (TOK_COLREF c c5)))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_COLREF a c1) c1) (TOK_SELEXPR (TOK_COLREF a c2) c2) (TOK_SELEXPR (TOK_COLREF b c3) c3) (TOK_SELEXPR (TOK_COLREF b c4) c4) (TOK_SELEXPR (TOK_COLREF c c5) c5) (TOK_SELEXPR (TOK_COLREF c c6) c6)))) c)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB dest1)) (TOK_SELECT (TOK_SELEXPR (TOK_COLREF c c1)) (TOK_SELEXPR (TOK_COLREF c c2)) (TOK_SELEXPR (TOK_COLREF c c3)) (TOK_SELEXPR (TOK_COLREF c c4)) (TOK_SELEXPR (TOK_COLREF c c5)) (TOK_SELEXPR (TOK_COLREF c c6))))) null \ No newline at end of file +(TOK_QUERY (TOK_FROM (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_LEFTOUTERJOIN (TOK_FULLOUTERJOIN (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF src src1)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_COLREF src1 key) c1) (TOK_SELEXPR (TOK_COLREF src1 value) c2)) (TOK_WHERE (and (> (TOK_COLREF src1 key) 10) (< (TOK_COLREF src1 key) 20))))) a) (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF src src2)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_COLREF src2 key) c3) (TOK_SELEXPR (TOK_COLREF src2 value) c4)) (TOK_WHERE (and (> (TOK_COLREF src2 key) 15) (< (TOK_COLREF src2 key) 25))))) b) (= (TOK_COLREF a c1) (TOK_COLREF b c3))) (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF src src3)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_COLREF src3 key) c5) (TOK_SELEXPR (TOK_COLREF src3 value) c6)) (TOK_WHERE (and (> (TOK_COLREF src3 key) 20) (< (TOK_COLREF src3 key) 25))))) c) (= (TOK_COLREF a c1) (TOK_COLREF c c5)))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_COLREF a c1) c1) (TOK_SELEXPR (TOK_COLREF a c2) c2) (TOK_SELEXPR (TOK_COLREF b c3) c3) (TOK_SELEXPR (TOK_COLREF b c4) c4) (TOK_SELEXPR (TOK_COLREF c c5) c5) (TOK_SELEXPR (TOK_COLREF c c6) c6)))) c)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_COLREF c c1)) (TOK_SELEXPR (TOK_COLREF c c2)) (TOK_SELEXPR (TOK_COLREF c c3)) (TOK_SELEXPR (TOK_COLREF c c4)) (TOK_SELEXPR (TOK_COLREF c c5)) (TOK_SELEXPR (TOK_COLREF c c6))))) null \ No newline at end of file Index: src/contrib/hive/ql/src/test/results/compiler/parse/input8.q.out =================================================================== --- src/contrib/hive/ql/src/test/results/compiler/parse/input8.q.out (revision 711472) +++ src/contrib/hive/ql/src/test/results/compiler/parse/input8.q.out (working copy) @@ -1 +1 @@ -(TOK_QUERY (TOK_FROM (TOK_TABREF src1)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB dest1)) (TOK_SELECT (TOK_SELEXPR (+ 4 TOK_NULL)) (TOK_SELEXPR (- (TOK_COLREF src1 key) TOK_NULL)) (TOK_SELEXPR (+ TOK_NULL TOK_NULL))))) null \ No newline at end of file +(TOK_QUERY (TOK_FROM (TOK_TABREF src1)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (+ 4 TOK_NULL)) (TOK_SELEXPR (- (TOK_COLREF src1 key) TOK_NULL)) (TOK_SELEXPR (+ TOK_NULL TOK_NULL))))) null \ No newline at end of file Index: src/contrib/hive/ql/src/test/results/compiler/parse/join8.q.out =================================================================== --- src/contrib/hive/ql/src/test/results/compiler/parse/join8.q.out (revision 711472) +++ src/contrib/hive/ql/src/test/results/compiler/parse/join8.q.out (working copy) @@ -1 +1 @@ -(TOK_QUERY (TOK_FROM (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_LEFTOUTERJOIN (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF src src1)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_COLREF src1 key) c1) (TOK_SELEXPR (TOK_COLREF src1 value) c2)) (TOK_WHERE (and (> (TOK_COLREF src1 key) 10) (< (TOK_COLREF src1 key) 20))))) a) (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF src src2)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_COLREF src2 key) c3) (TOK_SELEXPR (TOK_COLREF src2 value) c4)) (TOK_WHERE (and (> (TOK_COLREF src2 key) 15) (< (TOK_COLREF src2 key) 25))))) b) (= (TOK_COLREF a c1) (TOK_COLREF b c3)))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_COLREF a c1) c1) (TOK_SELEXPR (TOK_COLREF a c2) c2) (TOK_SELEXPR (TOK_COLREF b c3) c3) (TOK_SELEXPR (TOK_COLREF b c4) c4)))) c)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB dest1)) (TOK_SELECT (TOK_SELEXPR (TOK_COLREF c c1)) (TOK_SELEXPR (TOK_COLREF c c2)) (TOK_SELEXPR (TOK_COLREF c c3)) (TOK_SELEXPR (TOK_COLREF c c4))) (TOK_WHERE (AND (TOK_FUNCTION TOK_ISNULL (TOK_COLREF c c3)) (TOK_FUNCTION TOK_ISNOTNULL (TOK_COLREF c c1)))))) null \ No newline at end of file +(TOK_QUERY (TOK_FROM (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_LEFTOUTERJOIN (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF src src1)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_COLREF src1 key) c1) (TOK_SELEXPR (TOK_COLREF src1 value) c2)) (TOK_WHERE (and (> (TOK_COLREF src1 key) 10) (< (TOK_COLREF src1 key) 20))))) a) (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF src src2)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_COLREF src2 key) c3) (TOK_SELEXPR (TOK_COLREF src2 value) c4)) (TOK_WHERE (and (> (TOK_COLREF src2 key) 15) (< (TOK_COLREF src2 key) 25))))) b) (= (TOK_COLREF a c1) (TOK_COLREF b c3)))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_COLREF a c1) c1) (TOK_SELEXPR (TOK_COLREF a c2) c2) (TOK_SELEXPR (TOK_COLREF b c3) c3) (TOK_SELEXPR (TOK_COLREF b c4) c4)))) c)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_COLREF c c1)) (TOK_SELEXPR (TOK_COLREF c c2)) (TOK_SELEXPR (TOK_COLREF c c3)) (TOK_SELEXPR (TOK_COLREF c c4))) (TOK_WHERE (AND (TOK_FUNCTION TOK_ISNULL (TOK_COLREF c c3)) (TOK_FUNCTION TOK_ISNOTNULL (TOK_COLREF c c1)))))) null \ No newline at end of file Index: src/contrib/hive/ql/src/test/results/compiler/parse/udf1.q.out =================================================================== --- src/contrib/hive/ql/src/test/results/compiler/parse/udf1.q.out (revision 711472) +++ src/contrib/hive/ql/src/test/results/compiler/parse/udf1.q.out (working copy) @@ -1 +1 @@ -(TOK_QUERY (TOK_FROM (TOK_TABREF src)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB dest1)) (TOK_SELECT (TOK_SELEXPR (LIKE 'a' '%a%')) (TOK_SELEXPR (LIKE 'b' '%a%')) (TOK_SELEXPR (LIKE 'ab' '%a%')) (TOK_SELEXPR (LIKE 'ab' '%a_')) (TOK_SELEXPR (LIKE '%_' '\%\_')) (TOK_SELEXPR (LIKE 'ab' '\%\_')) (TOK_SELEXPR (LIKE 'ab' '_a%')) (TOK_SELEXPR (LIKE 'ab' 'a')) (TOK_SELEXPR (RLIKE '' '.*')) (TOK_SELEXPR (RLIKE 'a' '[ab]')) (TOK_SELEXPR (RLIKE '' '[ab]')) (TOK_SELEXPR (RLIKE 'hadoop' '[a-z]*')) (TOK_SELEXPR (RLIKE 'hadoop' 'o*')) (TOK_SELEXPR (TOK_FUNCTION REGEXP_REPLACE 'abc' 'b' 'c')) (TOK_SELEXPR (TOK_FUNCTION REGEXP_REPLACE 'abc' 'z' 'a')) (TOK_SELEXPR (TOK_FUNCTION REGEXP_REPLACE 'abbbb' 'bb' 'b')) (TOK_SELEXPR (TOK_FUNCTION REGEXP_REPLACE 'hadoop' '(.)[a-z]*' '$1ive'))) (TOK_WHERE (= (TOK_COLREF src key) 86)))) null \ No newline at end of file +(TOK_QUERY (TOK_FROM (TOK_TABREF src)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (LIKE 'a' '%a%')) (TOK_SELEXPR (LIKE 'b' '%a%')) (TOK_SELEXPR (LIKE 'ab' '%a%')) (TOK_SELEXPR (LIKE 'ab' '%a_')) (TOK_SELEXPR (LIKE '%_' '\%\_')) (TOK_SELEXPR (LIKE 'ab' '\%\_')) (TOK_SELEXPR (LIKE 'ab' '_a%')) (TOK_SELEXPR (LIKE 'ab' 'a')) (TOK_SELEXPR (RLIKE '' '.*')) (TOK_SELEXPR (RLIKE 'a' '[ab]')) (TOK_SELEXPR (RLIKE '' '[ab]')) (TOK_SELEXPR (RLIKE 'hadoop' '[a-z]*')) (TOK_SELEXPR (RLIKE 'hadoop' 'o*')) (TOK_SELEXPR (TOK_FUNCTION REGEXP_REPLACE 'abc' 'b' 'c')) (TOK_SELEXPR (TOK_FUNCTION REGEXP_REPLACE 'abc' 'z' 'a')) (TOK_SELEXPR (TOK_FUNCTION REGEXP_REPLACE 'abbbb' 'bb' 'b')) (TOK_SELEXPR (TOK_FUNCTION REGEXP_REPLACE 'hadoop' '(.)[a-z]*' '$1ive'))) (TOK_WHERE (= (TOK_COLREF src key) 86)))) null \ No newline at end of file Index: src/contrib/hive/ql/src/test/results/compiler/parse/input_testxpath.q.out =================================================================== --- src/contrib/hive/ql/src/test/results/compiler/parse/input_testxpath.q.out (revision 711472) +++ src/contrib/hive/ql/src/test/results/compiler/parse/input_testxpath.q.out (working copy) @@ -1 +1 @@ -(TOK_QUERY (TOK_FROM (TOK_TABREF src_thrift)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB dest1)) (TOK_SELECT (TOK_SELEXPR ([ (TOK_COLREF src_thrift lint) 1)) (TOK_SELEXPR (. ([ (TOK_COLREF src_thrift lintstring) 0) mystring)) (TOK_SELEXPR ([ (TOK_COLREF src_thrift mstringstring) 'key_2'))))) null \ No newline at end of file +(TOK_QUERY (TOK_FROM (TOK_TABREF src_thrift)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR ([ (TOK_COLREF src_thrift lint) 1)) (TOK_SELEXPR (. ([ (TOK_COLREF src_thrift lintstring) 0) mystring)) (TOK_SELEXPR ([ (TOK_COLREF src_thrift mstringstring) 'key_2'))))) null \ No newline at end of file Index: src/contrib/hive/ql/src/test/results/compiler/parse/input_part1.q.out =================================================================== --- src/contrib/hive/ql/src/test/results/compiler/parse/input_part1.q.out (revision 711472) +++ src/contrib/hive/ql/src/test/results/compiler/parse/input_part1.q.out (working copy) @@ -1 +1 @@ -(TOK_QUERY (TOK_FROM (TOK_TABREF srcpart)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB dest1)) (TOK_SELECT (TOK_SELEXPR (TOK_COLREF srcpart key)) (TOK_SELEXPR (TOK_COLREF srcpart value)) (TOK_SELEXPR (TOK_COLREF srcpart hr)) (TOK_SELEXPR (TOK_COLREF srcpart ds))) (TOK_WHERE (and (and (< (TOK_COLREF srcpart key) 100) (= (TOK_COLREF srcpart ds) '2008-04-08')) (= (TOK_COLREF srcpart hr) '12'))))) null \ No newline at end of file +(TOK_QUERY (TOK_FROM (TOK_TABREF srcpart)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_COLREF srcpart key)) (TOK_SELEXPR (TOK_COLREF srcpart value)) (TOK_SELEXPR (TOK_COLREF srcpart hr)) (TOK_SELEXPR (TOK_COLREF srcpart ds))) (TOK_WHERE (and (and (< (TOK_COLREF srcpart key) 100) (= (TOK_COLREF srcpart ds) '2008-04-08')) (= (TOK_COLREF srcpart hr) '12'))))) null \ No newline at end of file Index: src/contrib/hive/ql/src/test/results/compiler/parse/groupby2.q.out =================================================================== --- src/contrib/hive/ql/src/test/results/compiler/parse/groupby2.q.out (revision 711472) +++ src/contrib/hive/ql/src/test/results/compiler/parse/groupby2.q.out (working copy) @@ -1 +1 @@ -(TOK_QUERY (TOK_FROM (TOK_TABREF src)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB dest1)) (TOK_SELECT (TOK_SELEXPR (TOK_FUNCTION substr (TOK_COLREF src key) 0 1)) (TOK_SELEXPR (TOK_FUNCTIONDI count (TOK_FUNCTION substr (TOK_COLREF src value) 4))) (TOK_SELEXPR (TOK_FUNCTION concat (TOK_FUNCTION substr (TOK_COLREF src key) 0 1) (TOK_FUNCTION sum (TOK_FUNCTION substr (TOK_COLREF src value) 4))))) (TOK_GROUPBY (TOK_FUNCTION substr (TOK_COLREF src key) 0 1)))) null \ No newline at end of file +(TOK_QUERY (TOK_FROM (TOK_TABREF src)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_FUNCTION substr (TOK_COLREF src key) 0 1)) (TOK_SELEXPR (TOK_FUNCTIONDI count (TOK_FUNCTION substr (TOK_COLREF src value) 4))) (TOK_SELEXPR (TOK_FUNCTION concat (TOK_FUNCTION substr (TOK_COLREF src key) 0 1) (TOK_FUNCTION sum (TOK_FUNCTION substr (TOK_COLREF src value) 4))))) (TOK_GROUPBY (TOK_FUNCTION substr (TOK_COLREF src key) 0 1)))) null \ No newline at end of file Index: src/contrib/hive/ql/src/test/results/compiler/parse/groupby3.q.out =================================================================== --- src/contrib/hive/ql/src/test/results/compiler/parse/groupby3.q.out (revision 711472) +++ src/contrib/hive/ql/src/test/results/compiler/parse/groupby3.q.out (working copy) @@ -1 +1 @@ -(TOK_QUERY (TOK_FROM (TOK_TABREF src)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB dest1)) (TOK_SELECT (TOK_SELEXPR (TOK_FUNCTION sum (TOK_FUNCTION substr (TOK_COLREF src value) 4))) (TOK_SELEXPR (TOK_FUNCTION avg (TOK_FUNCTION substr (TOK_COLREF src value) 4))) (TOK_SELEXPR (TOK_FUNCTIONDI avg (TOK_FUNCTION substr (TOK_COLREF src value) 4))) (TOK_SELEXPR (TOK_FUNCTION max (TOK_FUNCTION substr (TOK_COLREF src value) 4))) (TOK_SELEXPR (TOK_FUNCTION min (TOK_FUNCTION substr (TOK_COLREF src value) 4)))))) null \ No newline at end of file +(TOK_QUERY (TOK_FROM (TOK_TABREF src)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_FUNCTION sum (TOK_FUNCTION substr (TOK_COLREF src value) 4))) (TOK_SELEXPR (TOK_FUNCTION avg (TOK_FUNCTION substr (TOK_COLREF src value) 4))) (TOK_SELEXPR (TOK_FUNCTIONDI avg (TOK_FUNCTION substr (TOK_COLREF src value) 4))) (TOK_SELEXPR (TOK_FUNCTION max (TOK_FUNCTION substr (TOK_COLREF src value) 4))) (TOK_SELEXPR (TOK_FUNCTION min (TOK_FUNCTION substr (TOK_COLREF src value) 4)))))) null \ No newline at end of file Index: src/contrib/hive/ql/src/test/results/compiler/parse/groupby4.q.out =================================================================== --- src/contrib/hive/ql/src/test/results/compiler/parse/groupby4.q.out (revision 711472) +++ src/contrib/hive/ql/src/test/results/compiler/parse/groupby4.q.out (working copy) @@ -1 +1 @@ -(TOK_QUERY (TOK_FROM (TOK_TABREF src)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB dest1)) (TOK_SELECT (TOK_SELEXPR (TOK_FUNCTION substr (TOK_COLREF src key) 0 1))) (TOK_GROUPBY (TOK_FUNCTION substr (TOK_COLREF src key) 0 1)))) null \ No newline at end of file +(TOK_QUERY (TOK_FROM (TOK_TABREF src)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_FUNCTION substr (TOK_COLREF src key) 0 1))) (TOK_GROUPBY (TOK_FUNCTION substr (TOK_COLREF src key) 0 1)))) null \ No newline at end of file Index: src/contrib/hive/ql/src/test/results/compiler/parse/groupby5.q.out =================================================================== --- src/contrib/hive/ql/src/test/results/compiler/parse/groupby5.q.out (revision 711472) +++ src/contrib/hive/ql/src/test/results/compiler/parse/groupby5.q.out (working copy) @@ -1 +1 @@ -(TOK_QUERY (TOK_FROM (TOK_TABREF src)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB dest1)) (TOK_SELECT (TOK_SELEXPR (TOK_COLREF src key)) (TOK_SELEXPR (TOK_FUNCTION sum (TOK_FUNCTION substr (TOK_COLREF src value) 4)))) (TOK_GROUPBY (TOK_COLREF src key)))) null \ No newline at end of file +(TOK_QUERY (TOK_FROM (TOK_TABREF src)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_COLREF src key)) (TOK_SELEXPR (TOK_FUNCTION sum (TOK_FUNCTION substr (TOK_COLREF src value) 4)))) (TOK_GROUPBY (TOK_COLREF src key)))) null \ No newline at end of file Index: src/contrib/hive/ql/src/test/results/compiler/parse/groupby6.q.out =================================================================== --- src/contrib/hive/ql/src/test/results/compiler/parse/groupby6.q.out (revision 711472) +++ src/contrib/hive/ql/src/test/results/compiler/parse/groupby6.q.out (working copy) @@ -1 +1 @@ -(TOK_QUERY (TOK_FROM (TOK_TABREF src)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB dest1)) (TOK_SELECTDI (TOK_SELEXPR (TOK_FUNCTION substr (TOK_COLREF src value) 4 1))))) null \ No newline at end of file +(TOK_QUERY (TOK_FROM (TOK_TABREF src)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECTDI (TOK_SELEXPR (TOK_FUNCTION substr (TOK_COLREF src value) 4 1))))) null \ No newline at end of file Index: src/contrib/hive/ql/src/test/results/compiler/parse/sample1.q.out =================================================================== --- src/contrib/hive/ql/src/test/results/compiler/parse/sample1.q.out (revision 711472) +++ src/contrib/hive/ql/src/test/results/compiler/parse/sample1.q.out (working copy) @@ -1 +1 @@ -(TOK_QUERY (TOK_FROM (TOK_TABREF srcpart (TOK_TABLESAMPLE 1 1) s)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB dest1)) (TOK_SELECT (TOK_SELEXPR (TOK_ALLCOLREF s))) (TOK_WHERE (and (= (TOK_COLREF s ds) '2008-04-08') (= (TOK_COLREF s hr) '11'))))) null \ No newline at end of file +(TOK_QUERY (TOK_FROM (TOK_TABREF srcpart (TOK_TABLESAMPLE 1 1 (TOK_FUNCTION rand)) s)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_ALLCOLREF s))) (TOK_WHERE (and (= (TOK_COLREF s ds) '2008-04-08') (= (TOK_COLREF s hr) '11'))))) null \ No newline at end of file Index: src/contrib/hive/ql/src/test/results/compiler/parse/sample3.q.out =================================================================== --- src/contrib/hive/ql/src/test/results/compiler/parse/sample3.q.out (revision 711472) +++ src/contrib/hive/ql/src/test/results/compiler/parse/sample3.q.out (working copy) @@ -1 +1 @@ -(TOK_QUERY (TOK_FROM (TOK_TABREF srcbucket (TOK_TABLESAMPLE 1 2 key value) s)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB dest1)) (TOK_SELECT (TOK_SELEXPR (TOK_ALLCOLREF s))))) null \ No newline at end of file +(TOK_QUERY (TOK_FROM (TOK_TABREF srcbucket (TOK_TABLESAMPLE 1 2 (TOK_COLREF key) (TOK_COLREF value)) s)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB dest1)) (TOK_SELECT (TOK_SELEXPR (TOK_ALLCOLREF s))))) null \ No newline at end of file Index: src/contrib/hive/ql/src/test/results/compiler/parse/sample4.q.out =================================================================== --- src/contrib/hive/ql/src/test/results/compiler/parse/sample4.q.out (revision 711472) +++ src/contrib/hive/ql/src/test/results/compiler/parse/sample4.q.out (working copy) @@ -1 +1 @@ -(TOK_QUERY (TOK_FROM (TOK_TABREF srcbucket (TOK_TABLESAMPLE 1 2 key) s)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB dest1)) (TOK_SELECT (TOK_SELEXPR (TOK_ALLCOLREF s))))) null \ No newline at end of file +(TOK_QUERY (TOK_FROM (TOK_TABREF srcbucket (TOK_TABLESAMPLE 1 2 (TOK_COLREF key)) s)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB dest1)) (TOK_SELECT (TOK_SELEXPR (TOK_ALLCOLREF s))))) null \ No newline at end of file Index: src/contrib/hive/ql/src/test/results/compiler/parse/sample5.q.out =================================================================== --- src/contrib/hive/ql/src/test/results/compiler/parse/sample5.q.out (revision 711472) +++ src/contrib/hive/ql/src/test/results/compiler/parse/sample5.q.out (working copy) @@ -1 +1 @@ -(TOK_QUERY (TOK_FROM (TOK_TABREF srcbucket (TOK_TABLESAMPLE 1 5 key) s)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB dest1)) (TOK_SELECT (TOK_SELEXPR (TOK_ALLCOLREF s))))) null \ No newline at end of file +(TOK_QUERY (TOK_FROM (TOK_TABREF srcbucket (TOK_TABLESAMPLE 1 5 (TOK_COLREF key)) s)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB dest1)) (TOK_SELECT (TOK_SELEXPR (TOK_ALLCOLREF s))))) null \ No newline at end of file Index: src/contrib/hive/ql/src/test/results/compiler/parse/sample6.q.out =================================================================== --- src/contrib/hive/ql/src/test/results/compiler/parse/sample6.q.out (revision 711472) +++ src/contrib/hive/ql/src/test/results/compiler/parse/sample6.q.out (working copy) @@ -1 +1 @@ -(TOK_QUERY (TOK_FROM (TOK_TABREF srcbucket (TOK_TABLESAMPLE 1 4 key) s)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB dest1)) (TOK_SELECT (TOK_SELEXPR (TOK_ALLCOLREF s))))) null \ No newline at end of file +(TOK_QUERY (TOK_FROM (TOK_TABREF srcbucket (TOK_TABLESAMPLE 1 4 (TOK_COLREF key)) s)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB dest1)) (TOK_SELECT (TOK_SELEXPR (TOK_ALLCOLREF s))))) null \ No newline at end of file Index: src/contrib/hive/ql/src/test/results/compiler/parse/sample7.q.out =================================================================== --- src/contrib/hive/ql/src/test/results/compiler/parse/sample7.q.out (revision 711472) +++ src/contrib/hive/ql/src/test/results/compiler/parse/sample7.q.out (working copy) @@ -1 +1 @@ -(TOK_QUERY (TOK_FROM (TOK_TABREF srcbucket (TOK_TABLESAMPLE 1 4 key) s)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB dest1)) (TOK_SELECT (TOK_SELEXPR (TOK_ALLCOLREF s))) (TOK_WHERE (> (TOK_COLREF s key) 100)))) null \ No newline at end of file +(TOK_QUERY (TOK_FROM (TOK_TABREF srcbucket (TOK_TABLESAMPLE 1 4 (TOK_COLREF key)) s)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB dest1)) (TOK_SELECT (TOK_SELEXPR (TOK_ALLCOLREF s))) (TOK_WHERE (> (TOK_COLREF s key) 100)))) null \ No newline at end of file Index: src/contrib/hive/ql/src/test/results/compiler/plan/join2.q.xml =================================================================== --- src/contrib/hive/ql/src/test/results/compiler/plan/join2.q.xml (revision 711472) +++ src/contrib/hive/ql/src/test/results/compiler/plan/join2.q.xml (working copy) @@ -38,7 +38,7 @@ true - /tmp/hive-zshao/1036144012/1033953173.10000.insclause-0 + /tmp/hive-zshao/645705020/257919804.10000.insclause-0 @@ -176,25 +176,29 @@ - + - org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe + org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe - org.apache.hadoop.mapred.TextInputFormat + org.apache.hadoop.mapred.SequenceFileInputFormat - org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + org.apache.hadoop.mapred.SequenceFileOutputFormat - columns - 0 + name + binary_sortable_table + serialization.ddl + struct binary_sortable_table { double joinkey0} + + serialization.format - 1 + org.apache.hadoop.hive.serde2.thrift.TBinarySortableProtocol @@ -234,25 +238,29 @@ - + - org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe + org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe - org.apache.hadoop.mapred.TextInputFormat + org.apache.hadoop.mapred.SequenceFileInputFormat - org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + org.apache.hadoop.mapred.SequenceFileOutputFormat - columns - 0,1 + name + binary_table + serialization.ddl + struct binary_table { string reducesinkvalue0, string reducesinkvalue1} + + serialization.format - 1 + com.facebook.thrift.protocol.TBinaryProtocol @@ -429,23 +437,27 @@ - org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe + org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe - org.apache.hadoop.mapred.TextInputFormat + org.apache.hadoop.mapred.SequenceFileInputFormat - org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + org.apache.hadoop.mapred.SequenceFileOutputFormat - columns - 0 + name + binary_sortable_table + serialization.ddl + struct binary_sortable_table { double joinkey0} + + serialization.format - 1 + org.apache.hadoop.hive.serde2.thrift.TBinarySortableProtocol @@ -502,25 +514,29 @@ - + - org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe + org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe - org.apache.hadoop.mapred.TextInputFormat + org.apache.hadoop.mapred.SequenceFileInputFormat - org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + org.apache.hadoop.mapred.SequenceFileOutputFormat - columns - 0,1,2,3 + name + binary_table + serialization.ddl + struct binary_table { string reducesinkvalue0, string reducesinkvalue1, string reducesinkvalue2, string reducesinkvalue3} + + serialization.format - 1 + com.facebook.thrift.protocol.TBinaryProtocol @@ -580,13 +596,16 @@ + + + true - /tmp/hive-zshao/1036144012/1033953173.10001 + /tmp/hive-zshao/645705020/257919804.10001 $INTNAME @@ -606,28 +625,32 @@ - /tmp/hive-zshao/1036144012/1033953173.10001 + /tmp/hive-zshao/645705020/257919804.10001 - + - org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe + org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe - org.apache.hadoop.mapred.TextInputFormat + org.apache.hadoop.mapred.SequenceFileInputFormat - org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + org.apache.hadoop.mapred.SequenceFileOutputFormat - columns - 0,1,2,3 + name + binary_table + serialization.ddl + struct binary_table { string temporarycol0, string temporarycol1, string temporarycol2, string temporarycol3} + + serialization.format - 1 + com.facebook.thrift.protocol.TBinaryProtocol @@ -714,7 +737,7 @@ - /tmp/hive-zshao/1036144012/1033953173.10000.insclause-0 + /tmp/hive-zshao/645705020/257919804.10000.insclause-0 @@ -951,6 +974,16 @@ + + + + + + + + + + @@ -993,23 +1026,27 @@ - org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe + org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe - org.apache.hadoop.mapred.TextInputFormat + org.apache.hadoop.mapred.SequenceFileInputFormat - org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + org.apache.hadoop.mapred.SequenceFileOutputFormat - columns - 0 + name + binary_sortable_table + serialization.ddl + struct binary_sortable_table { string joinkey0} + + serialization.format - 1 + org.apache.hadoop.hive.serde2.thrift.TBinarySortableProtocol @@ -1049,25 +1086,29 @@ - + - org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe + org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe - org.apache.hadoop.mapred.TextInputFormat + org.apache.hadoop.mapred.SequenceFileInputFormat - org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + org.apache.hadoop.mapred.SequenceFileOutputFormat - columns - 0,1 + name + binary_table + serialization.ddl + struct binary_table { string reducesinkvalue0, string reducesinkvalue1} + + serialization.format - 1 + com.facebook.thrift.protocol.TBinaryProtocol @@ -1161,25 +1202,29 @@ - + - org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe + org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe - org.apache.hadoop.mapred.TextInputFormat + org.apache.hadoop.mapred.SequenceFileInputFormat - org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + org.apache.hadoop.mapred.SequenceFileOutputFormat - columns - 0 + name + binary_sortable_table + serialization.ddl + struct binary_sortable_table { string joinkey0} + + serialization.format - 1 + org.apache.hadoop.hive.serde2.thrift.TBinarySortableProtocol @@ -1216,25 +1261,29 @@ - + - org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe + org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe - org.apache.hadoop.mapred.TextInputFormat + org.apache.hadoop.mapred.SequenceFileInputFormat - org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + org.apache.hadoop.mapred.SequenceFileOutputFormat - columns - 0,1 + name + binary_table + serialization.ddl + struct binary_table { string reducesinkvalue0, string reducesinkvalue1} + + serialization.format - 1 + com.facebook.thrift.protocol.TBinaryProtocol @@ -1306,6 +1355,9 @@ + + + true @@ -1401,10 +1453,10 @@ - /tmp/hive-zshao/1036144012/1033953173.10001 + /tmp/hive-zshao/645705020/257919804.10001 - + @@ -1534,6 +1586,16 @@ + + + + + + + + + + Index: src/contrib/hive/ql/src/test/results/compiler/plan/input2.q.xml =================================================================== --- src/contrib/hive/ql/src/test/results/compiler/plan/input2.q.xml (revision 711472) +++ src/contrib/hive/ql/src/test/results/compiler/plan/input2.q.xml (working copy) @@ -31,7 +31,7 @@ true - /tmp/hive-zshao/20345754/201293413.10000.insclause-0 + /tmp/hive-zshao/42137366/1420110739.10000.insclause-0 @@ -100,7 +100,7 @@ true - /tmp/hive-zshao/20345754/201293413.10001.insclause-1 + /tmp/hive-zshao/42137366/1420110739.10001.insclause-1 @@ -178,7 +178,7 @@ true - /tmp/hive-zshao/20345754/201293413.10002.insclause-2 + /tmp/hive-zshao/42137366/1420110739.10002.insclause-2 @@ -275,7 +275,7 @@ - /tmp/hive-zshao/20345754/201293413.10000.insclause-0 + /tmp/hive-zshao/42137366/1420110739.10000.insclause-0 @@ -459,7 +459,7 @@ - /tmp/hive-zshao/20345754/201293413.10001.insclause-1 + /tmp/hive-zshao/42137366/1420110739.10001.insclause-1 @@ -684,7 +684,7 @@ - /tmp/hive-zshao/20345754/201293413.10002.insclause-2 + /tmp/hive-zshao/42137366/1420110739.10002.insclause-2 @@ -705,6 +705,16 @@ + + + + 1 + + + + + + @@ -727,6 +737,16 @@ + + + + + + + 2 + + + @@ -895,6 +915,9 @@ + + + Index: src/contrib/hive/ql/src/test/results/compiler/plan/join3.q.xml =================================================================== --- src/contrib/hive/ql/src/test/results/compiler/plan/join3.q.xml (revision 711472) +++ src/contrib/hive/ql/src/test/results/compiler/plan/join3.q.xml (working copy) @@ -31,7 +31,7 @@ true - /tmp/hive-zshao/1679400869/64637247.10000.insclause-0 + /tmp/hive-zshao/56536666/678564191.10000.insclause-0 @@ -136,23 +136,27 @@ - org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe + org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe - org.apache.hadoop.mapred.TextInputFormat + org.apache.hadoop.mapred.SequenceFileInputFormat - org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + org.apache.hadoop.mapred.SequenceFileOutputFormat - columns - 0 + name + binary_sortable_table + serialization.ddl + struct binary_sortable_table { string joinkey0} + + serialization.format - 1 + org.apache.hadoop.hive.serde2.thrift.TBinarySortableProtocol @@ -192,25 +196,29 @@ - + - org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe + org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe - org.apache.hadoop.mapred.TextInputFormat + org.apache.hadoop.mapred.SequenceFileInputFormat - org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + org.apache.hadoop.mapred.SequenceFileOutputFormat - columns - 0,1 + name + binary_table + serialization.ddl + struct binary_table { string reducesinkvalue0, string reducesinkvalue1} + + serialization.format - 1 + com.facebook.thrift.protocol.TBinaryProtocol @@ -306,23 +314,27 @@ - org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe + org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe - org.apache.hadoop.mapred.TextInputFormat + org.apache.hadoop.mapred.SequenceFileInputFormat - org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + org.apache.hadoop.mapred.SequenceFileOutputFormat - columns - 0 + name + binary_sortable_table + serialization.ddl + struct binary_sortable_table { string joinkey0} + + serialization.format - 1 + org.apache.hadoop.hive.serde2.thrift.TBinarySortableProtocol @@ -362,25 +374,29 @@ - + - org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe + org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe - org.apache.hadoop.mapred.TextInputFormat + org.apache.hadoop.mapred.SequenceFileInputFormat - org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + org.apache.hadoop.mapred.SequenceFileOutputFormat - columns - 0,1 + name + binary_table + serialization.ddl + struct binary_table { string reducesinkvalue0, string reducesinkvalue1} + + serialization.format - 1 + com.facebook.thrift.protocol.TBinaryProtocol @@ -474,25 +490,29 @@ - + - org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe + org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe - org.apache.hadoop.mapred.TextInputFormat + org.apache.hadoop.mapred.SequenceFileInputFormat - org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + org.apache.hadoop.mapred.SequenceFileOutputFormat - columns - 0 + name + binary_sortable_table + serialization.ddl + struct binary_sortable_table { string joinkey0} + + serialization.format - 1 + org.apache.hadoop.hive.serde2.thrift.TBinarySortableProtocol @@ -529,25 +549,29 @@ - + - org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe + org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe - org.apache.hadoop.mapred.TextInputFormat + org.apache.hadoop.mapred.SequenceFileInputFormat - org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + org.apache.hadoop.mapred.SequenceFileOutputFormat - columns - 0,1 + name + binary_table + serialization.ddl + struct binary_table { string reducesinkvalue0, string reducesinkvalue1} + + serialization.format - 1 + com.facebook.thrift.protocol.TBinaryProtocol @@ -619,6 +643,9 @@ + + + true @@ -721,7 +748,7 @@ - /tmp/hive-zshao/1679400869/64637247.10000.insclause-0 + /tmp/hive-zshao/56536666/678564191.10000.insclause-0 @@ -970,6 +997,19 @@ + + + + + + + + + + + + + Index: src/contrib/hive/ql/src/test/results/compiler/plan/input3.q.xml =================================================================== --- src/contrib/hive/ql/src/test/results/compiler/plan/input3.q.xml (revision 711472) +++ src/contrib/hive/ql/src/test/results/compiler/plan/input3.q.xml (working copy) @@ -28,7 +28,7 @@ true - /tmp/hive-njain/356693511/91381909.10003.insclause-3 + /tmp/hive-zshao/372733769/1189566131.10003.insclause-3 ../../../../build/contrib/hive/ql/test/data/warehouse/dest4.out @@ -48,7 +48,7 @@ true - /tmp/hive-njain/356693511/91381909.10000.insclause-0 + /tmp/hive-zshao/372733769/1189566131.10000.insclause-0 @@ -97,7 +97,7 @@ location - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/dest1 + file:/data/users/zshao/tubbs-svnroot/projects/hadoop/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/dest1 @@ -117,7 +117,7 @@ true - /tmp/hive-njain/356693511/91381909.10001.insclause-1 + /tmp/hive-zshao/372733769/1189566131.10001.insclause-1 @@ -166,7 +166,7 @@ location - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/dest2 + file:/data/users/zshao/tubbs-svnroot/projects/hadoop/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/dest2 @@ -195,7 +195,7 @@ true - /tmp/hive-njain/356693511/91381909.10002.insclause-2 + /tmp/hive-zshao/372733769/1189566131.10002.insclause-2 @@ -248,7 +248,7 @@ location - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/dest3 + file:/data/users/zshao/tubbs-svnroot/projects/hadoop/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/dest3 @@ -292,7 +292,7 @@ - /tmp/hive-njain/356693511/91381909.10000.insclause-0 + /tmp/hive-zshao/372733769/1189566131.10000.insclause-0 @@ -476,7 +476,7 @@ - /tmp/hive-njain/356693511/91381909.10001.insclause-1 + /tmp/hive-zshao/372733769/1189566131.10001.insclause-1 @@ -701,7 +701,7 @@ - /tmp/hive-njain/356693511/91381909.10002.insclause-2 + /tmp/hive-zshao/372733769/1189566131.10002.insclause-2 @@ -722,6 +722,16 @@ + + + + 1 + + + + + + @@ -744,6 +754,16 @@ + + + + + + + 2 + + + @@ -906,7 +926,7 @@ - /tmp/hive-njain/356693511/91381909.10003.insclause-3 + /tmp/hive-zshao/372733769/1189566131.10003.insclause-3 @@ -922,6 +942,10 @@ + columns + value + + serialization.format 1 @@ -1058,7 +1082,7 @@ - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/src + file:/data/users/zshao/tubbs-svnroot/projects/hadoop/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/src src @@ -1070,7 +1094,7 @@ - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/src + file:/data/users/zshao/tubbs-svnroot/projects/hadoop/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/src @@ -1122,7 +1146,7 @@ location - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/src + file:/data/users/zshao/tubbs-svnroot/projects/hadoop/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/src @@ -1135,6 +1159,9 @@ + + + Index: src/contrib/hive/ql/src/test/results/compiler/plan/join4.q.xml =================================================================== --- src/contrib/hive/ql/src/test/results/compiler/plan/join4.q.xml (revision 711472) +++ src/contrib/hive/ql/src/test/results/compiler/plan/join4.q.xml (working copy) @@ -1,104 +1,6 @@ - - - - - - - Stage-1 - - - - - - - - - - - - - - - - - - - - - - true - - - /tmp/hive-zshao/257975023/155084340.10000.insclause-0 - - - - - org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe - - - org.apache.hadoop.mapred.TextInputFormat - - - org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat - - - - - name - dest1 - - - serialization.ddl - struct dest1 { string key, string value} - - - serialization.format - 1 - - - columns - key,value - - - bucket_count - -1 - - - serialization.lib - org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe - - - file.inputformat - org.apache.hadoop.mapred.TextInputFormat - - - file.outputformat - org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat - - - location - file:/data/users/zshao/tubbs-svnroot/projects/hadoop/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/dest1 - - - - - org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe - - - - - - - - - - - - - + Stage-2 @@ -142,25 +44,29 @@ - + - org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe + org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe - org.apache.hadoop.mapred.TextInputFormat + org.apache.hadoop.mapred.SequenceFileInputFormat - org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + org.apache.hadoop.mapred.SequenceFileOutputFormat - columns - 0 + name + binary_sortable_table + serialization.ddl + struct binary_sortable_table { string joinkey0} + + serialization.format - 1 + org.apache.hadoop.hive.serde2.thrift.TBinarySortableProtocol @@ -200,25 +106,29 @@ - + - org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe + org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe - org.apache.hadoop.mapred.TextInputFormat + org.apache.hadoop.mapred.SequenceFileInputFormat - org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + org.apache.hadoop.mapred.SequenceFileOutputFormat - columns - 0,1 + name + binary_table + serialization.ddl + struct binary_table { string reducesinkvalue0, string reducesinkvalue1} + + serialization.format - 1 + com.facebook.thrift.protocol.TBinaryProtocol @@ -526,23 +436,27 @@ - org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe + org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe - org.apache.hadoop.mapred.TextInputFormat + org.apache.hadoop.mapred.SequenceFileInputFormat - org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + org.apache.hadoop.mapred.SequenceFileOutputFormat - columns - 0 + name + binary_sortable_table + serialization.ddl + struct binary_sortable_table { string joinkey0} + + serialization.format - 1 + org.apache.hadoop.hive.serde2.thrift.TBinarySortableProtocol @@ -579,25 +493,29 @@ - + - org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe + org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe - org.apache.hadoop.mapred.TextInputFormat + org.apache.hadoop.mapred.SequenceFileInputFormat - org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + org.apache.hadoop.mapred.SequenceFileOutputFormat - columns - 0,1 + name + binary_table + serialization.ddl + struct binary_table { string reducesinkvalue0, string reducesinkvalue1} + + serialization.format - 1 + com.facebook.thrift.protocol.TBinaryProtocol @@ -865,6 +783,9 @@ + + + true @@ -968,10 +889,32 @@ - /tmp/hive-zshao/257975023/155084340.10000.insclause-0 + /tmp/hive-zshao/321830556/108856276.10001.insclause-0 - + + + org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe + + + org.apache.hadoop.mapred.TextInputFormat + + + org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + + + + + columns + c1,c2,c3,c4 + + + serialization.format + 1 + + + + @@ -1308,6 +1251,16 @@ + + + + + + + + + + Index: src/contrib/hive/ql/src/test/results/compiler/plan/input4.q.xml =================================================================== --- src/contrib/hive/ql/src/test/results/compiler/plan/input4.q.xml (revision 711472) +++ src/contrib/hive/ql/src/test/results/compiler/plan/input4.q.xml (working copy) @@ -31,7 +31,7 @@ true - /tmp/hive-zshao/1042895489/270462051.10000.insclause-0 + /tmp/hive-zshao/1248673405/435834590.10000.insclause-0 @@ -142,25 +142,29 @@ - + - org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe + org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe - org.apache.hadoop.mapred.TextInputFormat + org.apache.hadoop.mapred.SequenceFileInputFormat - org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + org.apache.hadoop.mapred.SequenceFileOutputFormat - columns - 0 + name + binary_sortable_table + serialization.ddl + struct binary_sortable_table { string reducesinkkey0} + + serialization.format - 1 + org.apache.hadoop.hive.serde2.thrift.TBinarySortableProtocol @@ -200,25 +204,29 @@ - + - org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe + org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe - org.apache.hadoop.mapred.TextInputFormat + org.apache.hadoop.mapred.SequenceFileInputFormat - org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + org.apache.hadoop.mapred.SequenceFileOutputFormat - columns - 0,1 + name + binary_table + serialization.ddl + struct binary_table { string reducesinkvalue0, string reducesinkvalue1} + + serialization.format - 1 + com.facebook.thrift.protocol.TBinaryProtocol @@ -418,6 +426,9 @@ + + + @@ -515,7 +526,7 @@ - /tmp/hive-zshao/1042895489/270462051.10000.insclause-0 + /tmp/hive-zshao/1248673405/435834590.10000.insclause-0 @@ -707,6 +718,13 @@ + + + + + + + Index: src/contrib/hive/ql/src/test/results/compiler/plan/join5.q.xml =================================================================== --- src/contrib/hive/ql/src/test/results/compiler/plan/join5.q.xml (revision 711472) +++ src/contrib/hive/ql/src/test/results/compiler/plan/join5.q.xml (working copy) @@ -1,104 +1,6 @@ - - - - - - - Stage-1 - - - - - - - - - - - - - - - - - - - - - - true - - - /tmp/hive-zshao/936351131/313796179.10000.insclause-0 - - - - - org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe - - - org.apache.hadoop.mapred.TextInputFormat - - - org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat - - - - - name - dest1 - - - serialization.ddl - struct dest1 { string key, string value} - - - serialization.format - 1 - - - columns - key,value - - - bucket_count - -1 - - - serialization.lib - org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe - - - file.inputformat - org.apache.hadoop.mapred.TextInputFormat - - - file.outputformat - org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat - - - location - file:/data/users/zshao/tubbs-svnroot/projects/hadoop/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/dest1 - - - - - org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe - - - - - - - - - - - - - + Stage-2 @@ -142,25 +44,29 @@ - + - org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe + org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe - org.apache.hadoop.mapred.TextInputFormat + org.apache.hadoop.mapred.SequenceFileInputFormat - org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + org.apache.hadoop.mapred.SequenceFileOutputFormat - columns - 0 + name + binary_sortable_table + serialization.ddl + struct binary_sortable_table { string joinkey0} + + serialization.format - 1 + org.apache.hadoop.hive.serde2.thrift.TBinarySortableProtocol @@ -200,25 +106,29 @@ - + - org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe + org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe - org.apache.hadoop.mapred.TextInputFormat + org.apache.hadoop.mapred.SequenceFileInputFormat - org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + org.apache.hadoop.mapred.SequenceFileOutputFormat - columns - 0,1 + name + binary_table + serialization.ddl + struct binary_table { string reducesinkvalue0, string reducesinkvalue1} + + serialization.format - 1 + com.facebook.thrift.protocol.TBinaryProtocol @@ -526,23 +436,27 @@ - org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe + org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe - org.apache.hadoop.mapred.TextInputFormat + org.apache.hadoop.mapred.SequenceFileInputFormat - org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + org.apache.hadoop.mapred.SequenceFileOutputFormat - columns - 0 + name + binary_sortable_table + serialization.ddl + struct binary_sortable_table { string joinkey0} + + serialization.format - 1 + org.apache.hadoop.hive.serde2.thrift.TBinarySortableProtocol @@ -579,25 +493,29 @@ - + - org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe + org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe - org.apache.hadoop.mapred.TextInputFormat + org.apache.hadoop.mapred.SequenceFileInputFormat - org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + org.apache.hadoop.mapred.SequenceFileOutputFormat - columns - 0,1 + name + binary_table + serialization.ddl + struct binary_table { string reducesinkvalue0, string reducesinkvalue1} + + serialization.format - 1 + com.facebook.thrift.protocol.TBinaryProtocol @@ -865,6 +783,9 @@ + + + true @@ -968,10 +889,32 @@ - /tmp/hive-zshao/936351131/313796179.10000.insclause-0 + /tmp/hive-zshao/647376731/102582680.10001.insclause-0 - + + + org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe + + + org.apache.hadoop.mapred.TextInputFormat + + + org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + + + + + columns + c1,c2,c3,c4 + + + serialization.format + 1 + + + + @@ -1308,6 +1251,16 @@ + + + + + + + + + + Index: src/contrib/hive/ql/src/test/results/compiler/plan/input5.q.xml =================================================================== --- src/contrib/hive/ql/src/test/results/compiler/plan/input5.q.xml (revision 711472) +++ src/contrib/hive/ql/src/test/results/compiler/plan/input5.q.xml (working copy) @@ -31,7 +31,7 @@ true - /tmp/hive-zshao/641253778/15454385.10000.insclause-0 + /tmp/hive-zshao/245100626/667810226.10000.insclause-0 @@ -142,25 +142,29 @@ - + - org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe + org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe - org.apache.hadoop.mapred.TextInputFormat + org.apache.hadoop.mapred.SequenceFileInputFormat - org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + org.apache.hadoop.mapred.SequenceFileOutputFormat - columns - 0 + name + binary_sortable_table + serialization.ddl + struct binary_sortable_table { string reducesinkkey0} + + serialization.format - 1 + org.apache.hadoop.hive.serde2.thrift.TBinarySortableProtocol @@ -200,25 +204,29 @@ - + - org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe + org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe - org.apache.hadoop.mapred.TextInputFormat + org.apache.hadoop.mapred.SequenceFileInputFormat - org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + org.apache.hadoop.mapred.SequenceFileOutputFormat - columns - 0,1 + name + binary_table + serialization.ddl + struct binary_table { string reducesinkvalue0, string reducesinkvalue1} + + serialization.format - 1 + com.facebook.thrift.protocol.TBinaryProtocol @@ -481,6 +489,9 @@ + + + @@ -578,7 +589,7 @@ - /tmp/hive-zshao/641253778/15454385.10000.insclause-0 + /tmp/hive-zshao/245100626/667810226.10000.insclause-0 @@ -700,6 +711,13 @@ + + + + + + + Index: src/contrib/hive/ql/src/test/results/compiler/plan/join6.q.xml =================================================================== --- src/contrib/hive/ql/src/test/results/compiler/plan/join6.q.xml (revision 711472) +++ src/contrib/hive/ql/src/test/results/compiler/plan/join6.q.xml (working copy) @@ -1,104 +1,6 @@ - - - - - - - Stage-1 - - - - - - - - - - - - - - - - - - - - - - true - - - /tmp/hive-zshao/161126421/564619381.10000.insclause-0 - - - - - org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe - - - org.apache.hadoop.mapred.TextInputFormat - - - org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat - - - - - name - dest1 - - - serialization.ddl - struct dest1 { string key, string value} - - - serialization.format - 1 - - - columns - key,value - - - bucket_count - -1 - - - serialization.lib - org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe - - - file.inputformat - org.apache.hadoop.mapred.TextInputFormat - - - file.outputformat - org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat - - - location - file:/data/users/zshao/tubbs-svnroot/projects/hadoop/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/dest1 - - - - - org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe - - - - - - - - - - - - - + Stage-2 @@ -142,25 +44,29 @@ - + - org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe + org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe - org.apache.hadoop.mapred.TextInputFormat + org.apache.hadoop.mapred.SequenceFileInputFormat - org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + org.apache.hadoop.mapred.SequenceFileOutputFormat - columns - 0 + name + binary_sortable_table + serialization.ddl + struct binary_sortable_table { string joinkey0} + + serialization.format - 1 + org.apache.hadoop.hive.serde2.thrift.TBinarySortableProtocol @@ -200,25 +106,29 @@ - + - org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe + org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe - org.apache.hadoop.mapred.TextInputFormat + org.apache.hadoop.mapred.SequenceFileInputFormat - org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + org.apache.hadoop.mapred.SequenceFileOutputFormat - columns - 0,1 + name + binary_table + serialization.ddl + struct binary_table { string reducesinkvalue0, string reducesinkvalue1} + + serialization.format - 1 + com.facebook.thrift.protocol.TBinaryProtocol @@ -526,23 +436,27 @@ - org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe + org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe - org.apache.hadoop.mapred.TextInputFormat + org.apache.hadoop.mapred.SequenceFileInputFormat - org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + org.apache.hadoop.mapred.SequenceFileOutputFormat - columns - 0 + name + binary_sortable_table + serialization.ddl + struct binary_sortable_table { string joinkey0} + + serialization.format - 1 + org.apache.hadoop.hive.serde2.thrift.TBinarySortableProtocol @@ -579,25 +493,29 @@ - + - org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe + org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe - org.apache.hadoop.mapred.TextInputFormat + org.apache.hadoop.mapred.SequenceFileInputFormat - org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + org.apache.hadoop.mapred.SequenceFileOutputFormat - columns - 0,1 + name + binary_table + serialization.ddl + struct binary_table { string reducesinkvalue0, string reducesinkvalue1} + + serialization.format - 1 + com.facebook.thrift.protocol.TBinaryProtocol @@ -865,6 +783,9 @@ + + + true @@ -968,10 +889,32 @@ - /tmp/hive-zshao/161126421/564619381.10000.insclause-0 + /tmp/hive-zshao/495982087/695855038.10001.insclause-0 - + + + org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe + + + org.apache.hadoop.mapred.TextInputFormat + + + org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + + + + + columns + c1,c2,c3,c4 + + + serialization.format + 1 + + + + @@ -1308,6 +1251,16 @@ + + + + + + + + + + Index: src/contrib/hive/ql/src/test/results/compiler/plan/input_testxpath2.q.xml =================================================================== --- src/contrib/hive/ql/src/test/results/compiler/plan/input_testxpath2.q.xml (revision 711472) +++ src/contrib/hive/ql/src/test/results/compiler/plan/input_testxpath2.q.xml (working copy) @@ -1,104 +1,6 @@ - - - - - - - Stage-1 - - - - - - - - - - - - - - - - - - - - - - true - - - /tmp/hive-zshao/421781212/46322741.10000.insclause-0 - - - - - org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe - - - org.apache.hadoop.mapred.TextInputFormat - - - org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat - - - - - name - dest1 - - - serialization.ddl - struct dest1 { string key, string value} - - - serialization.format - 1 - - - columns - key,value - - - bucket_count - -1 - - - serialization.lib - org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe - - - file.inputformat - org.apache.hadoop.mapred.TextInputFormat - - - file.outputformat - org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat - - - location - file:/data/users/zshao/tubbs-svnroot/projects/hadoop/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/dest1 - - - - - org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe - - - - - - - - - - - - - + Stage-2 @@ -124,10 +26,32 @@ - /tmp/hive-zshao/421781212/46322741.10000.insclause-0 + /tmp/hive-zshao/283385472/835267807.10001.insclause-0 - + + + org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe + + + org.apache.hadoop.mapred.TextInputFormat + + + org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + + + + + columns + _c0,_c1,_c2 + + + serialization.format + 1 + + + + @@ -615,6 +539,9 @@ + + + Index: src/contrib/hive/ql/src/test/results/compiler/plan/input6.q.xml =================================================================== --- src/contrib/hive/ql/src/test/results/compiler/plan/input6.q.xml (revision 711472) +++ src/contrib/hive/ql/src/test/results/compiler/plan/input6.q.xml (working copy) @@ -31,7 +31,7 @@ true - /tmp/hive-zshao/301573619/12587577.10000.insclause-0 + /tmp/hive-zshao/8144642/99473545.10000.insclause-0 @@ -124,7 +124,7 @@ - /tmp/hive-zshao/301573619/12587577.10000.insclause-0 + /tmp/hive-zshao/8144642/99473545.10000.insclause-0 @@ -371,6 +371,9 @@ + + + Index: src/contrib/hive/ql/src/test/results/compiler/plan/join7.q.xml =================================================================== --- src/contrib/hive/ql/src/test/results/compiler/plan/join7.q.xml (revision 711472) +++ src/contrib/hive/ql/src/test/results/compiler/plan/join7.q.xml (working copy) @@ -1,104 +1,6 @@ - - - - - - - Stage-1 - - - - - - - - - - - - - - - - - - - - - - true - - - /tmp/hive-zshao/61032244/444662007.10000.insclause-0 - - - - - org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe - - - org.apache.hadoop.mapred.TextInputFormat - - - org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat - - - - - name - dest1 - - - serialization.ddl - struct dest1 { string key, string value} - - - serialization.format - 1 - - - columns - key,value - - - bucket_count - -1 - - - serialization.lib - org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe - - - file.inputformat - org.apache.hadoop.mapred.TextInputFormat - - - file.outputformat - org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat - - - location - file:/data/users/zshao/tubbs-svnroot/projects/hadoop/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/dest1 - - - - - org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe - - - - - - - - - - - - - + Stage-2 @@ -144,23 +46,27 @@ - org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe + org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe - org.apache.hadoop.mapred.TextInputFormat + org.apache.hadoop.mapred.SequenceFileInputFormat - org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + org.apache.hadoop.mapred.SequenceFileOutputFormat - columns - 0 + name + binary_sortable_table + serialization.ddl + struct binary_sortable_table { string joinkey0} + + serialization.format - 1 + org.apache.hadoop.hive.serde2.thrift.TBinarySortableProtocol @@ -200,25 +106,29 @@ - + - org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe + org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe - org.apache.hadoop.mapred.TextInputFormat + org.apache.hadoop.mapred.SequenceFileInputFormat - org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + org.apache.hadoop.mapred.SequenceFileOutputFormat - columns - 0,1 + name + binary_table + serialization.ddl + struct binary_table { string reducesinkvalue0, string reducesinkvalue1} + + serialization.format - 1 + com.facebook.thrift.protocol.TBinaryProtocol @@ -526,23 +436,27 @@ - org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe + org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe - org.apache.hadoop.mapred.TextInputFormat + org.apache.hadoop.mapred.SequenceFileInputFormat - org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + org.apache.hadoop.mapred.SequenceFileOutputFormat - columns - 0 + name + binary_sortable_table + serialization.ddl + struct binary_sortable_table { string joinkey0} + + serialization.format - 1 + org.apache.hadoop.hive.serde2.thrift.TBinarySortableProtocol @@ -579,25 +493,29 @@ - + - org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe + org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe - org.apache.hadoop.mapred.TextInputFormat + org.apache.hadoop.mapred.SequenceFileInputFormat - org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + org.apache.hadoop.mapred.SequenceFileOutputFormat - columns - 0,1 + name + binary_table + serialization.ddl + struct binary_table { string reducesinkvalue0, string reducesinkvalue1} + + serialization.format - 1 + com.facebook.thrift.protocol.TBinaryProtocol @@ -895,25 +813,29 @@ - + - org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe + org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe - org.apache.hadoop.mapred.TextInputFormat + org.apache.hadoop.mapred.SequenceFileInputFormat - org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + org.apache.hadoop.mapred.SequenceFileOutputFormat - columns - 0 + name + binary_sortable_table + serialization.ddl + struct binary_sortable_table { string joinkey0} + + serialization.format - 1 + org.apache.hadoop.hive.serde2.thrift.TBinarySortableProtocol @@ -953,25 +875,29 @@ - + - org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe + org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe - org.apache.hadoop.mapred.TextInputFormat + org.apache.hadoop.mapred.SequenceFileInputFormat - org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + org.apache.hadoop.mapred.SequenceFileOutputFormat - columns - 0,1 + name + binary_table + serialization.ddl + struct binary_table { string reducesinkvalue0, string reducesinkvalue1} + + serialization.format - 1 + com.facebook.thrift.protocol.TBinaryProtocol @@ -1239,6 +1165,9 @@ + + + true @@ -1345,10 +1274,32 @@ - /tmp/hive-zshao/61032244/444662007.10000.insclause-0 + /tmp/hive-zshao/752275695/363016745.10001.insclause-0 - + + + org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe + + + org.apache.hadoop.mapred.TextInputFormat + + + org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + + + + + columns + c1,c2,c3,c4,c5,c6 + + + serialization.format + 1 + + + + @@ -1820,6 +1771,19 @@ + + + + + + + + + + + + + Index: src/contrib/hive/ql/src/test/results/compiler/plan/input7.q.xml =================================================================== --- src/contrib/hive/ql/src/test/results/compiler/plan/input7.q.xml (revision 711472) +++ src/contrib/hive/ql/src/test/results/compiler/plan/input7.q.xml (working copy) @@ -31,7 +31,7 @@ true - /tmp/hive-zshao/182607732/631207979.10000.insclause-0 + /tmp/hive-zshao/397972505/393019179.10000.insclause-0 @@ -120,7 +120,7 @@ - /tmp/hive-zshao/182607732/631207979.10000.insclause-0 + /tmp/hive-zshao/397972505/393019179.10000.insclause-0 @@ -315,6 +315,9 @@ + + + Index: src/contrib/hive/ql/src/test/results/compiler/plan/input8.q.xml =================================================================== --- src/contrib/hive/ql/src/test/results/compiler/plan/input8.q.xml (revision 711472) +++ src/contrib/hive/ql/src/test/results/compiler/plan/input8.q.xml (working copy) @@ -1,104 +1,6 @@ - - - - - - - Stage-1 - - - - - - - - - - - - - - - - - - - - - - true - - - /tmp/hive-zshao/243084011/767326882.10000.insclause-0 - - - - - org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe - - - org.apache.hadoop.mapred.TextInputFormat - - - org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat - - - - - name - dest1 - - - serialization.ddl - struct dest1 { string key, string value} - - - serialization.format - 1 - - - columns - key,value - - - bucket_count - -1 - - - serialization.lib - org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe - - - file.inputformat - org.apache.hadoop.mapred.TextInputFormat - - - file.outputformat - org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat - - - location - file:/data/users/zshao/tubbs-svnroot/projects/hadoop/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/dest1 - - - - - org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe - - - - - - - - - - - - - + Stage-2 @@ -120,10 +22,32 @@ - /tmp/hive-zshao/243084011/767326882.10000.insclause-0 + /tmp/hive-zshao/566113162/356511991.10001.insclause-0 - + + + org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe + + + org.apache.hadoop.mapred.TextInputFormat + + + org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + + + + + columns + _c0,_c1,_c2 + + + serialization.format + 1 + + + + @@ -469,6 +393,9 @@ + + + Index: src/contrib/hive/ql/src/test/results/compiler/plan/join8.q.xml =================================================================== --- src/contrib/hive/ql/src/test/results/compiler/plan/join8.q.xml (revision 711472) +++ src/contrib/hive/ql/src/test/results/compiler/plan/join8.q.xml (working copy) @@ -1,104 +1,6 @@ - - - - - - - Stage-1 - - - - - - - - - - - - - - - - - - - - - - true - - - /tmp/hive-zshao/616083721/271603255.10000.insclause-0 - - - - - org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe - - - org.apache.hadoop.mapred.TextInputFormat - - - org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat - - - - - name - dest1 - - - serialization.ddl - struct dest1 { string key, string value} - - - serialization.format - 1 - - - columns - key,value - - - bucket_count - -1 - - - serialization.lib - org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe - - - file.inputformat - org.apache.hadoop.mapred.TextInputFormat - - - file.outputformat - org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat - - - location - file:/data/users/zshao/tubbs-svnroot/projects/hadoop/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/dest1 - - - - - org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe - - - - - - - - - - - - - + Stage-2 @@ -142,25 +44,29 @@ - + - org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe + org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe - org.apache.hadoop.mapred.TextInputFormat + org.apache.hadoop.mapred.SequenceFileInputFormat - org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + org.apache.hadoop.mapred.SequenceFileOutputFormat - columns - 0 + name + binary_sortable_table + serialization.ddl + struct binary_sortable_table { string joinkey0} + + serialization.format - 1 + org.apache.hadoop.hive.serde2.thrift.TBinarySortableProtocol @@ -200,25 +106,29 @@ - + - org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe + org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe - org.apache.hadoop.mapred.TextInputFormat + org.apache.hadoop.mapred.SequenceFileInputFormat - org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + org.apache.hadoop.mapred.SequenceFileOutputFormat - columns - 0,1 + name + binary_table + serialization.ddl + struct binary_table { string reducesinkvalue0, string reducesinkvalue1} + + serialization.format - 1 + com.facebook.thrift.protocol.TBinaryProtocol @@ -526,23 +436,27 @@ - org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe + org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe - org.apache.hadoop.mapred.TextInputFormat + org.apache.hadoop.mapred.SequenceFileInputFormat - org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + org.apache.hadoop.mapred.SequenceFileOutputFormat - columns - 0 + name + binary_sortable_table + serialization.ddl + struct binary_sortable_table { string joinkey0} + + serialization.format - 1 + org.apache.hadoop.hive.serde2.thrift.TBinarySortableProtocol @@ -579,25 +493,29 @@ - + - org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe + org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe - org.apache.hadoop.mapred.TextInputFormat + org.apache.hadoop.mapred.SequenceFileInputFormat - org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + org.apache.hadoop.mapred.SequenceFileOutputFormat - columns - 0,1 + name + binary_table + serialization.ddl + struct binary_table { string reducesinkvalue0, string reducesinkvalue1} + + serialization.format - 1 + com.facebook.thrift.protocol.TBinaryProtocol @@ -865,6 +783,9 @@ + + + true @@ -972,10 +893,32 @@ - /tmp/hive-zshao/616083721/271603255.10000.insclause-0 + /tmp/hive-zshao/176847532/59223613.10001.insclause-0 - + + + org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe + + + org.apache.hadoop.mapred.TextInputFormat + + + org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + + + + + columns + c1,c2,c3,c4 + + + serialization.format + 1 + + + + @@ -1422,6 +1365,16 @@ + + + + + + + + + + Index: src/contrib/hive/ql/src/test/results/compiler/plan/input_testsequencefile.q.xml =================================================================== --- src/contrib/hive/ql/src/test/results/compiler/plan/input_testsequencefile.q.xml (revision 711472) +++ src/contrib/hive/ql/src/test/results/compiler/plan/input_testsequencefile.q.xml (working copy) @@ -31,7 +31,7 @@ true - /tmp/hive-zshao/1185059341/85219314.10000.insclause-0 + /tmp/hive-zshao/165610669/11698833.10000.insclause-0 @@ -120,7 +120,7 @@ - /tmp/hive-zshao/1185059341/85219314.10000.insclause-0 + /tmp/hive-zshao/165610669/11698833.10000.insclause-0 @@ -314,6 +314,9 @@ + + + Index: src/contrib/hive/ql/src/test/results/compiler/plan/union.q.xml =================================================================== --- src/contrib/hive/ql/src/test/results/compiler/plan/union.q.xml (revision 711472) +++ src/contrib/hive/ql/src/test/results/compiler/plan/union.q.xml (working copy) @@ -28,7 +28,7 @@ true - /tmp/hive-njain/635791418/469963157.10000.insclause-0 + /tmp/hive-zshao/74629618/612066829.10000.insclause-0 ../../../../build/contrib/hive/ql/test/data/warehouse/union.out @@ -79,7 +79,7 @@ - /tmp/hive-njain/635791418/469963157.10000.insclause-0 + /tmp/hive-zshao/74629618/612066829.10000.insclause-0 @@ -95,6 +95,10 @@ + columns + key,value + + serialization.format 1 @@ -528,7 +532,7 @@ - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/src + file:/data/users/zshao/tubbs-svnroot/projects/hadoop/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/src null-subquery1:unioninput-subquery1:src @@ -543,7 +547,7 @@ - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/src + file:/data/users/zshao/tubbs-svnroot/projects/hadoop/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/src @@ -595,7 +599,7 @@ location - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/src + file:/data/users/zshao/tubbs-svnroot/projects/hadoop/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/src @@ -608,6 +612,9 @@ + + + Index: src/contrib/hive/ql/src/test/results/compiler/plan/input9.q.xml =================================================================== --- src/contrib/hive/ql/src/test/results/compiler/plan/input9.q.xml (revision 711472) +++ src/contrib/hive/ql/src/test/results/compiler/plan/input9.q.xml (working copy) @@ -31,7 +31,7 @@ true - /tmp/hive-zshao/1031500531/592905830.10000.insclause-0 + /tmp/hive-zshao/1191638521/210802858.10000.insclause-0 @@ -124,7 +124,7 @@ - /tmp/hive-zshao/1031500531/592905830.10000.insclause-0 + /tmp/hive-zshao/1191638521/210802858.10000.insclause-0 @@ -379,6 +379,9 @@ + + + Index: src/contrib/hive/ql/src/test/results/compiler/plan/udf1.q.xml =================================================================== --- src/contrib/hive/ql/src/test/results/compiler/plan/udf1.q.xml (revision 711472) +++ src/contrib/hive/ql/src/test/results/compiler/plan/udf1.q.xml (working copy) @@ -1,104 +1,6 @@ - - - - - - - Stage-1 - - - - - - - - - - - - - - - - - - - - - - true - - - /tmp/hive-zshao/247949494/314100641.10000.insclause-0 - - - - - org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe - - - org.apache.hadoop.mapred.TextInputFormat - - - org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat - - - - - name - dest1 - - - serialization.ddl - struct dest1 { string key, string value} - - - serialization.format - 1 - - - columns - key,value - - - bucket_count - -1 - - - serialization.lib - org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe - - - file.inputformat - org.apache.hadoop.mapred.TextInputFormat - - - file.outputformat - org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat - - - location - file:/data/users/zshao/tubbs-svnroot/projects/hadoop/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/dest1 - - - - - org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe - - - - - - - - - - - - - + Stage-2 @@ -124,10 +26,32 @@ - /tmp/hive-zshao/247949494/314100641.10000.insclause-0 + /tmp/hive-zshao/174508321/372293.10001.insclause-0 - + + + org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe + + + org.apache.hadoop.mapred.TextInputFormat + + + org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + + + + + columns + _c0,_c1,_c2,_c3,_c4,_c5,_c6,_c7,_c8,_c9,_c10,_c11,_c12,_c13,_c14,_c15,_c16 + + + serialization.format + 1 + + + + @@ -1369,6 +1293,9 @@ + + + Index: src/contrib/hive/ql/src/test/results/compiler/plan/udf4.q.xml =================================================================== --- src/contrib/hive/ql/src/test/results/compiler/plan/udf4.q.xml (revision 711472) +++ src/contrib/hive/ql/src/test/results/compiler/plan/udf4.q.xml (working copy) @@ -22,7 +22,7 @@ - /tmp/hive-njain/463574005/202732636.10001.insclause-0 + /tmp/hive-zshao/8775795/346367389.10001.insclause-0 @@ -38,6 +38,10 @@ + columns + _c0,_c1,_c2,_c3,_c4,_c5,_c6,_c7,_c8,_c9,_c10,_c11,_c12,_c13,_c14,_c15 + + serialization.format 1 @@ -955,7 +959,7 @@ - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/dest1 + file:/data/users/zshao/tubbs-svnroot/projects/hadoop/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/dest1 dest1 @@ -967,7 +971,7 @@ - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/dest1 + file:/data/users/zshao/tubbs-svnroot/projects/hadoop/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/dest1 @@ -1019,7 +1023,7 @@ location - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/dest1 + file:/data/users/zshao/tubbs-svnroot/projects/hadoop/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/dest1 @@ -1032,6 +1036,9 @@ + + + Index: src/contrib/hive/ql/src/test/results/compiler/plan/input_testxpath.q.xml =================================================================== --- src/contrib/hive/ql/src/test/results/compiler/plan/input_testxpath.q.xml (revision 711472) +++ src/contrib/hive/ql/src/test/results/compiler/plan/input_testxpath.q.xml (working copy) @@ -1,104 +1,6 @@ - - - - - - - Stage-1 - - - - - - - - - - - - - - - - - - - - - - true - - - /tmp/hive-zshao/780638111/964614278.10000.insclause-0 - - - - - org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe - - - org.apache.hadoop.mapred.TextInputFormat - - - org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat - - - - - name - dest1 - - - serialization.ddl - struct dest1 { string key, string value} - - - serialization.format - 1 - - - columns - key,value - - - bucket_count - -1 - - - serialization.lib - org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe - - - file.inputformat - org.apache.hadoop.mapred.TextInputFormat - - - file.outputformat - org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat - - - location - file:/data/users/zshao/tubbs-svnroot/projects/hadoop/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/dest1 - - - - - org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe - - - - - - - - - - - - - + Stage-2 @@ -120,10 +22,32 @@ - /tmp/hive-zshao/780638111/964614278.10000.insclause-0 + /tmp/hive-zshao/5482975/920076456.10001.insclause-0 - + + + org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe + + + org.apache.hadoop.mapred.TextInputFormat + + + org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + + + + + columns + _c0,mystring,_c2 + + + serialization.format + 1 + + + + @@ -465,6 +389,9 @@ + + + Index: src/contrib/hive/ql/src/test/results/compiler/plan/input_part1.q.xml =================================================================== --- src/contrib/hive/ql/src/test/results/compiler/plan/input_part1.q.xml (revision 711472) +++ src/contrib/hive/ql/src/test/results/compiler/plan/input_part1.q.xml (working copy) @@ -1,104 +1,6 @@ - - - - - - - Stage-1 - - - - - - - - - - - - - - - - - - - - - - true - - - /tmp/hive-zshao/587528285/609899583.10000.insclause-0 - - - - - org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe - - - org.apache.hadoop.mapred.TextInputFormat - - - org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat - - - - - name - dest1 - - - serialization.ddl - struct dest1 { string key, string value} - - - serialization.format - 1 - - - columns - key,value - - - bucket_count - -1 - - - serialization.lib - org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe - - - file.inputformat - org.apache.hadoop.mapred.TextInputFormat - - - file.outputformat - org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat - - - location - file:/data/users/zshao/tubbs-svnroot/projects/hadoop/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/dest1 - - - - - org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe - - - - - - - - - - - - - + Stage-2 @@ -124,10 +26,32 @@ - /tmp/hive-zshao/587528285/609899583.10000.insclause-0 + /tmp/hive-zshao/470811431/33402790.10001.insclause-0 - + + + org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe + + + org.apache.hadoop.mapred.TextInputFormat + + + org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + + + + + columns + key,value,hr,ds + + + serialization.format + 1 + + + + @@ -609,6 +533,9 @@ + + + Index: src/contrib/hive/ql/src/test/results/compiler/plan/groupby1.q.xml =================================================================== --- src/contrib/hive/ql/src/test/results/compiler/plan/groupby1.q.xml (revision 711472) +++ src/contrib/hive/ql/src/test/results/compiler/plan/groupby1.q.xml (working copy) @@ -35,7 +35,7 @@ true - /tmp/hive-zshao/1211327466/29184745.10000.insclause-0 + /tmp/hive-zshao/522036038/904625919.10000.insclause-0 @@ -118,7 +118,7 @@ - /tmp/hive-zshao/1211327466/29184745.10001 + /tmp/hive-zshao/522036038/904625919.10001 @@ -144,25 +144,29 @@ - + - org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe + org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe - org.apache.hadoop.mapred.TextInputFormat + org.apache.hadoop.mapred.SequenceFileInputFormat - org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + org.apache.hadoop.mapred.SequenceFileOutputFormat - columns - 0 + name + binary_sortable_table + serialization.ddl + struct binary_sortable_table { string reducesinkkey0} + + serialization.format - 1 + org.apache.hadoop.hive.serde2.thrift.TBinarySortableProtocol @@ -185,32 +189,40 @@ 1 - + + + java.lang.Double + + - + - org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe + org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe - org.apache.hadoop.mapred.TextInputFormat + org.apache.hadoop.mapred.SequenceFileInputFormat - org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + org.apache.hadoop.mapred.SequenceFileOutputFormat - columns - 0 + name + binary_table + serialization.ddl + struct binary_table { double reducesinkvalue0} + + serialization.format - 1 + com.facebook.thrift.protocol.TBinaryProtocol @@ -238,7 +250,7 @@ VALUE.0 - + @@ -253,13 +265,16 @@ true + + + - /tmp/hive-zshao/1211327466/29184745.10001 + /tmp/hive-zshao/522036038/904625919.10001 - /tmp/hive-zshao/1211327466/29184745.10001 + /tmp/hive-zshao/522036038/904625919.10001 @@ -268,28 +283,32 @@ - /tmp/hive-zshao/1211327466/29184745.10001 + /tmp/hive-zshao/522036038/904625919.10001 - + - org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe + org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe - org.apache.hadoop.mapred.TextInputFormat + org.apache.hadoop.mapred.SequenceFileInputFormat - org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + org.apache.hadoop.mapred.SequenceFileOutputFormat - columns - 0,1 + name + binary_table + serialization.ddl + struct binary_table { string temporarycol0, double temporarycol1} + + serialization.format - 1 + com.facebook.thrift.protocol.TBinaryProtocol @@ -312,7 +331,7 @@ - /tmp/hive-zshao/1211327466/29184745.10000.insclause-0 + /tmp/hive-zshao/522036038/904625919.10000.insclause-0 @@ -339,7 +358,7 @@ 1 - + @@ -371,7 +390,7 @@ 1 - + @@ -407,7 +426,7 @@ VALUE.0 - + @@ -433,7 +452,7 @@ - PARTIAL2 + FINAL @@ -458,7 +477,7 @@ 1 - + @@ -468,6 +487,13 @@ + + + + + + + @@ -505,25 +531,29 @@ - + - org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe + org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe - org.apache.hadoop.mapred.TextInputFormat + org.apache.hadoop.mapred.SequenceFileInputFormat - org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + org.apache.hadoop.mapred.SequenceFileOutputFormat - columns - 0 + name + binary_sortable_table + serialization.ddl + struct binary_sortable_table { string reducesinkkey0} + + serialization.format - 1 + org.apache.hadoop.hive.serde2.thrift.TBinarySortableProtocol @@ -594,25 +624,29 @@ - + - org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe + org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe - org.apache.hadoop.mapred.TextInputFormat + org.apache.hadoop.mapred.SequenceFileInputFormat - org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + org.apache.hadoop.mapred.SequenceFileOutputFormat - columns - 0 + name + binary_table + serialization.ddl + struct binary_table { string reducesinkvalue0} + + serialization.format - 1 + com.facebook.thrift.protocol.TBinaryProtocol @@ -684,6 +718,9 @@ + + + @@ -773,10 +810,10 @@ - /tmp/hive-zshao/1211327466/29184745.10001 + /tmp/hive-zshao/522036038/904625919.10001 - + @@ -800,7 +837,7 @@ 1 - + @@ -824,12 +861,36 @@ - - - VALUE.0 + + + org.apache.hadoop.hive.ql.udf.UDFToDouble + + + evaluate + + + java.lang.String + + + + + + + + + + VALUE.0 + + + + + + + + - + @@ -865,6 +926,13 @@ + + + + + + + Index: src/contrib/hive/ql/src/test/results/compiler/plan/groupby2.q.xml =================================================================== --- src/contrib/hive/ql/src/test/results/compiler/plan/groupby2.q.xml (revision 711472) +++ src/contrib/hive/ql/src/test/results/compiler/plan/groupby2.q.xml (working copy) @@ -4,105 +4,7 @@ - - - - - - - Stage-1 - - - - - - - - - - - - - - - - - - - - - - true - - - /tmp/hive-zshao/62629504/152527911.10000.insclause-0 - - - - - org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe - - - org.apache.hadoop.mapred.TextInputFormat - - - org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat - - - - - name - dest1 - - - serialization.ddl - struct dest1 { string key, string value} - - - serialization.format - 1 - - - columns - key,value - - - bucket_count - -1 - - - serialization.lib - org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe - - - file.inputformat - org.apache.hadoop.mapred.TextInputFormat - - - file.outputformat - org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat - - - location - file:/data/users/zshao/tubbs-svnroot/projects/hadoop/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/dest1 - - - - - org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe - - - - - - - - - - - - - + Stage-3 @@ -118,7 +20,7 @@ - /tmp/hive-zshao/62629504/152527911.10001 + /tmp/hive-zshao/2920989/501793534.10002 @@ -144,25 +46,29 @@ - + - org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe + org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe - org.apache.hadoop.mapred.TextInputFormat + org.apache.hadoop.mapred.SequenceFileInputFormat - org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + org.apache.hadoop.mapred.SequenceFileOutputFormat - columns - 0 + name + binary_sortable_table + serialization.ddl + struct binary_sortable_table { string reducesinkkey0} + + serialization.format - 1 + org.apache.hadoop.hive.serde2.thrift.TBinarySortableProtocol @@ -185,7 +91,11 @@ 1 - + + + java.lang.Long + + @@ -195,32 +105,40 @@ 2 - + + + java.lang.Double + + - + - org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe + org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe - org.apache.hadoop.mapred.TextInputFormat + org.apache.hadoop.mapred.SequenceFileInputFormat - org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + org.apache.hadoop.mapred.SequenceFileOutputFormat - columns - 0,1 + name + binary_table + serialization.ddl + struct binary_table { i64 reducesinkvalue0, double reducesinkvalue1} + + serialization.format - 1 + com.facebook.thrift.protocol.TBinaryProtocol @@ -248,7 +166,7 @@ VALUE.0 - + @@ -258,7 +176,7 @@ VALUE.1 - + @@ -273,13 +191,16 @@ true + + + - /tmp/hive-zshao/62629504/152527911.10001 + /tmp/hive-zshao/2920989/501793534.10002 - /tmp/hive-zshao/62629504/152527911.10001 + /tmp/hive-zshao/2920989/501793534.10002 @@ -288,28 +209,32 @@ - /tmp/hive-zshao/62629504/152527911.10001 + /tmp/hive-zshao/2920989/501793534.10002 - + - org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe + org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe - org.apache.hadoop.mapred.TextInputFormat + org.apache.hadoop.mapred.SequenceFileInputFormat - org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + org.apache.hadoop.mapred.SequenceFileOutputFormat - columns - 0,1,2 + name + binary_table + serialization.ddl + struct binary_table { string temporarycol0, i64 temporarycol1, double temporarycol2} + + serialization.format - 1 + com.facebook.thrift.protocol.TBinaryProtocol @@ -332,10 +257,32 @@ - /tmp/hive-zshao/62629504/152527911.10000.insclause-0 + /tmp/hive-zshao/2920989/501793534.10001.insclause-0 - + + + org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe + + + org.apache.hadoop.mapred.TextInputFormat + + + org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + + + + + columns + _c0,_c1,_c2 + + + serialization.format + 1 + + + + @@ -359,7 +306,7 @@ 1 - + @@ -401,7 +348,7 @@ 1 - + @@ -436,10 +383,34 @@ - - - 2 + + + org.apache.hadoop.hive.ql.udf.UDFToString + + + evaluate + + + java.lang.Double + + + + + + + + + + 2 + + + + + + + + @@ -484,7 +455,7 @@ VALUE.0 - + @@ -505,7 +476,7 @@ VALUE.1 - + @@ -531,7 +502,7 @@ - PARTIAL2 + FINAL @@ -556,7 +527,7 @@ 1 - + @@ -566,7 +537,7 @@ 2 - + @@ -576,6 +547,13 @@ + + + + + + + @@ -636,7 +614,7 @@ - + java.lang.Integer @@ -650,7 +628,7 @@ - + 1 @@ -697,7 +675,7 @@ - + 4 @@ -714,25 +692,29 @@ - + - org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe + org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe - org.apache.hadoop.mapred.TextInputFormat + org.apache.hadoop.mapred.SequenceFileInputFormat - org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + org.apache.hadoop.mapred.SequenceFileOutputFormat - columns - 0,1 + name + binary_sortable_table + serialization.ddl + struct binary_sortable_table { string reducesinkkey0, string reducesinkkey1} + + serialization.format - 1 + org.apache.hadoop.hive.serde2.thrift.TBinarySortableProtocol @@ -751,25 +733,29 @@ - + - org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe + org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe - org.apache.hadoop.mapred.TextInputFormat + org.apache.hadoop.mapred.SequenceFileInputFormat - org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + org.apache.hadoop.mapred.SequenceFileOutputFormat - columns - + name + binary_table + serialization.ddl + struct binary_table { } + + serialization.format - 1 + com.facebook.thrift.protocol.TBinaryProtocol @@ -841,6 +827,9 @@ + + + @@ -930,10 +919,10 @@ - /tmp/hive-zshao/62629504/152527911.10001 + /tmp/hive-zshao/2920989/501793534.10002 - + @@ -957,7 +946,7 @@ 1 - + @@ -967,7 +956,7 @@ 2 - + @@ -1015,12 +1004,36 @@ - - - KEY.1 + + + org.apache.hadoop.hive.ql.udf.UDFToDouble + + + evaluate + + + java.lang.String + + + + + + + + + + KEY.1 + + + + + + + + - + @@ -1056,6 +1069,13 @@ + + + + + + + Index: src/contrib/hive/ql/src/test/results/compiler/plan/subq.q.xml =================================================================== --- src/contrib/hive/ql/src/test/results/compiler/plan/subq.q.xml (revision 711472) +++ src/contrib/hive/ql/src/test/results/compiler/plan/subq.q.xml (working copy) @@ -28,7 +28,7 @@ true - /tmp/hive-njain/560916151/1453534343.10000.insclause-0 + /tmp/hive-zshao/99536515/780288562.10000.insclause-0 ../../../../build/contrib/hive/ql/test/data/warehouse/union.out @@ -75,7 +75,7 @@ - /tmp/hive-njain/560916151/1453534343.10000.insclause-0 + /tmp/hive-zshao/99536515/780288562.10000.insclause-0 @@ -91,6 +91,10 @@ + columns + key,value + + serialization.format 1 @@ -340,7 +344,7 @@ - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/src + file:/data/users/zshao/tubbs-svnroot/projects/hadoop/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/src unioninput:src @@ -352,7 +356,7 @@ - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/src + file:/data/users/zshao/tubbs-svnroot/projects/hadoop/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/src @@ -404,7 +408,7 @@ location - file:/home/njain/workspace/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/src + file:/data/users/zshao/tubbs-svnroot/projects/hadoop/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/src @@ -417,6 +421,9 @@ + + + Index: src/contrib/hive/ql/src/test/results/compiler/plan/groupby3.q.xml =================================================================== --- src/contrib/hive/ql/src/test/results/compiler/plan/groupby3.q.xml (revision 711472) +++ src/contrib/hive/ql/src/test/results/compiler/plan/groupby3.q.xml (working copy) @@ -4,105 +4,7 @@ - - - - - - - Stage-1 - - - - - - - - - - - - - - - - - - - - - - true - - - /tmp/hive-zshao/168733871/318815163.10000.insclause-0 - - - - - org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe - - - org.apache.hadoop.mapred.TextInputFormat - - - org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat - - - - - name - dest1 - - - serialization.ddl - struct dest1 { string key, string value} - - - serialization.format - 1 - - - columns - key,value - - - bucket_count - -1 - - - serialization.lib - org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe - - - file.inputformat - org.apache.hadoop.mapred.TextInputFormat - - - file.outputformat - org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat - - - location - file:/data/users/zshao/tubbs-svnroot/projects/hadoop/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/dest1 - - - - - org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe - - - - - - - - - - - - - + Stage-3 @@ -118,7 +20,7 @@ - /tmp/hive-zshao/168733871/318815163.10001 + /tmp/hive-zshao/870051013/39742963.10002 @@ -129,25 +31,29 @@ - + - org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe + org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe - org.apache.hadoop.mapred.TextInputFormat + org.apache.hadoop.mapred.SequenceFileInputFormat - org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + org.apache.hadoop.mapred.SequenceFileOutputFormat - columns - + name + binary_sortable_table + serialization.ddl + struct binary_sortable_table { } + + serialization.format - 1 + org.apache.hadoop.hive.serde2.thrift.TBinarySortableProtocol @@ -181,7 +87,11 @@ 1 - + + + java.lang.Double + + @@ -201,7 +111,7 @@ 3 - + @@ -211,32 +121,36 @@ 4 - + - + - org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe + org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe - org.apache.hadoop.mapred.TextInputFormat + org.apache.hadoop.mapred.SequenceFileInputFormat - org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + org.apache.hadoop.mapred.SequenceFileOutputFormat - columns - 0,1,2,3,4 + name + binary_table + serialization.ddl + struct binary_table { string reducesinkvalue0, double reducesinkvalue1, string reducesinkvalue2, double reducesinkvalue3, double reducesinkvalue4} + + serialization.format - 1 + com.facebook.thrift.protocol.TBinaryProtocol @@ -264,7 +178,7 @@ VALUE.1 - + @@ -284,7 +198,7 @@ VALUE.3 - + @@ -294,7 +208,7 @@ VALUE.4 - + @@ -309,13 +223,16 @@ true + + + - /tmp/hive-zshao/168733871/318815163.10001 + /tmp/hive-zshao/870051013/39742963.10002 - /tmp/hive-zshao/168733871/318815163.10001 + /tmp/hive-zshao/870051013/39742963.10002 @@ -324,28 +241,32 @@ - /tmp/hive-zshao/168733871/318815163.10001 + /tmp/hive-zshao/870051013/39742963.10002 - + - org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe + org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe - org.apache.hadoop.mapred.TextInputFormat + org.apache.hadoop.mapred.SequenceFileInputFormat - org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + org.apache.hadoop.mapred.SequenceFileOutputFormat - columns - 0,1,2,3,4 + name + binary_table + serialization.ddl + struct binary_table { string temporarycol0, double temporarycol1, string temporarycol2, double temporarycol3, double temporarycol4} + + serialization.format - 1 + com.facebook.thrift.protocol.TBinaryProtocol @@ -368,10 +289,32 @@ - /tmp/hive-zshao/168733871/318815163.10000.insclause-0 + /tmp/hive-zshao/870051013/39742963.10001.insclause-0 - + + + org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe + + + org.apache.hadoop.mapred.TextInputFormat + + + org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + + + + + columns + _c0,_c1,_c2,_c3,_c4 + + + serialization.format + 1 + + + + @@ -385,7 +328,7 @@ 0 - + @@ -415,7 +358,7 @@ 3 - + @@ -425,7 +368,7 @@ 4 - + @@ -447,7 +390,7 @@ 1 - + @@ -477,7 +420,7 @@ 4 - + @@ -487,7 +430,7 @@ 3 - + @@ -544,7 +487,7 @@ VALUE.1 - + @@ -586,7 +529,7 @@ VALUE.3 - + @@ -607,7 +550,7 @@ VALUE.4 - + @@ -622,7 +565,7 @@ - PARTIAL2 + FINAL @@ -647,7 +590,7 @@ 1 - + @@ -667,7 +610,7 @@ 3 - + @@ -677,7 +620,7 @@ 4 - + @@ -687,6 +630,13 @@ + + + + + + + @@ -765,25 +715,29 @@ - + - org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe + org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe - org.apache.hadoop.mapred.TextInputFormat + org.apache.hadoop.mapred.SequenceFileInputFormat - org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + org.apache.hadoop.mapred.SequenceFileOutputFormat - columns - 0 + name + binary_sortable_table + serialization.ddl + struct binary_sortable_table { string reducesinkkey0} + + serialization.format - 1 + org.apache.hadoop.hive.serde2.thrift.TBinarySortableProtocol @@ -802,25 +756,29 @@ - + - org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe + org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe - org.apache.hadoop.mapred.TextInputFormat + org.apache.hadoop.mapred.SequenceFileInputFormat - org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + org.apache.hadoop.mapred.SequenceFileOutputFormat - columns - + name + binary_table + serialization.ddl + struct binary_table { } + + serialization.format - 1 + com.facebook.thrift.protocol.TBinaryProtocol @@ -882,6 +840,9 @@ + + + @@ -971,10 +932,10 @@ - /tmp/hive-zshao/168733871/318815163.10001 + /tmp/hive-zshao/870051013/39742963.10002 - + @@ -998,7 +959,7 @@ 1 - + @@ -1018,7 +979,7 @@ 3 - + @@ -1028,7 +989,7 @@ 4 - + @@ -1055,12 +1016,36 @@ - - - KEY.0 + + + org.apache.hadoop.hive.ql.udf.UDFToDouble + + + evaluate + + + java.lang.String + + + + + + + + + + KEY.0 + + + + + + + + - + @@ -1076,12 +1061,36 @@ - - - KEY.0 + + + org.apache.hadoop.hive.ql.udf.UDFToDouble + + + evaluate + + + java.lang.String + + + + + + + + + + KEY.0 + + + + + + + + - + @@ -1097,12 +1106,36 @@ - - - KEY.0 + + + org.apache.hadoop.hive.ql.udf.UDFToDouble + + + evaluate + + + java.lang.String + + + + + + + + + + KEY.0 + + + + + + + + - + @@ -1118,12 +1151,36 @@ - - - KEY.0 + + + org.apache.hadoop.hive.ql.udf.UDFToDouble + + + evaluate + + + java.lang.String + + + + + + + + + + KEY.0 + + + + + + + + - + @@ -1139,12 +1196,36 @@ - - - KEY.0 + + + org.apache.hadoop.hive.ql.udf.UDFToDouble + + + evaluate + + + java.lang.String + + + + + + + + + + KEY.0 + + + + + + + + - + @@ -1169,6 +1250,13 @@ + + + + + + + Index: src/contrib/hive/ql/src/test/results/compiler/plan/groupby4.q.xml =================================================================== --- src/contrib/hive/ql/src/test/results/compiler/plan/groupby4.q.xml (revision 711472) +++ src/contrib/hive/ql/src/test/results/compiler/plan/groupby4.q.xml (working copy) @@ -4,105 +4,7 @@ - - - - - - - Stage-1 - - - - - - - - - - - - - - - - - - - - - - true - - - /tmp/hive-zshao/843671827/38838214.10000.insclause-0 - - - - - org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe - - - org.apache.hadoop.mapred.TextInputFormat - - - org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat - - - - - name - dest1 - - - serialization.ddl - struct dest1 { string key, string value} - - - serialization.format - 1 - - - columns - key,value - - - bucket_count - -1 - - - serialization.lib - org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe - - - file.inputformat - org.apache.hadoop.mapred.TextInputFormat - - - file.outputformat - org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat - - - location - file:/data/users/zshao/tubbs-svnroot/projects/hadoop/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/dest1 - - - - - org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe - - - - - - - - - - - - - + Stage-3 @@ -118,7 +20,7 @@ - /tmp/hive-zshao/843671827/38838214.10001 + /tmp/hive-zshao/180131506/43353471.10002 @@ -144,25 +46,29 @@ - + - org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe + org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe - org.apache.hadoop.mapred.TextInputFormat + org.apache.hadoop.mapred.SequenceFileInputFormat - org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + org.apache.hadoop.mapred.SequenceFileOutputFormat - columns - 0 + name + binary_sortable_table + serialization.ddl + struct binary_sortable_table { string reducesinkkey0} + + serialization.format - 1 + org.apache.hadoop.hive.serde2.thrift.TBinarySortableProtocol @@ -181,25 +87,29 @@ - + - org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe + org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe - org.apache.hadoop.mapred.TextInputFormat + org.apache.hadoop.mapred.SequenceFileInputFormat - org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + org.apache.hadoop.mapred.SequenceFileOutputFormat - columns - + name + binary_table + serialization.ddl + struct binary_table { } + + serialization.format - 1 + com.facebook.thrift.protocol.TBinaryProtocol @@ -232,13 +142,16 @@ true + + + - /tmp/hive-zshao/843671827/38838214.10001 + /tmp/hive-zshao/180131506/43353471.10002 - /tmp/hive-zshao/843671827/38838214.10001 + /tmp/hive-zshao/180131506/43353471.10002 @@ -247,28 +160,32 @@ - /tmp/hive-zshao/843671827/38838214.10001 + /tmp/hive-zshao/180131506/43353471.10002 - + - org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe + org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe - org.apache.hadoop.mapred.TextInputFormat + org.apache.hadoop.mapred.SequenceFileInputFormat - org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + org.apache.hadoop.mapred.SequenceFileOutputFormat - columns - 0 + name + binary_table + serialization.ddl + struct binary_table { string temporarycol0} + + serialization.format - 1 + com.facebook.thrift.protocol.TBinaryProtocol @@ -291,10 +208,32 @@ - /tmp/hive-zshao/843671827/38838214.10000.insclause-0 + /tmp/hive-zshao/180131506/43353471.10001.insclause-0 - + + + org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe + + + org.apache.hadoop.mapred.TextInputFormat + + + org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + + + + + columns + _c0 + + + serialization.format + 1 + + + + @@ -370,7 +309,7 @@ - PARTIAL2 + FINAL @@ -395,6 +334,13 @@ + + + + + + + @@ -486,25 +432,29 @@ - + - org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe + org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe - org.apache.hadoop.mapred.TextInputFormat + org.apache.hadoop.mapred.SequenceFileInputFormat - org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + org.apache.hadoop.mapred.SequenceFileOutputFormat - columns - 0 + name + binary_sortable_table + serialization.ddl + struct binary_sortable_table { string reducesinkkey0} + + serialization.format - 1 + org.apache.hadoop.hive.serde2.thrift.TBinarySortableProtocol @@ -523,25 +473,29 @@ - + - org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe + org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe - org.apache.hadoop.mapred.TextInputFormat + org.apache.hadoop.mapred.SequenceFileInputFormat - org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + org.apache.hadoop.mapred.SequenceFileOutputFormat - columns - + name + binary_table + serialization.ddl + struct binary_table { } + + serialization.format - 1 + com.facebook.thrift.protocol.TBinaryProtocol @@ -603,6 +557,9 @@ + + + @@ -692,10 +649,10 @@ - /tmp/hive-zshao/843671827/38838214.10001 + /tmp/hive-zshao/180131506/43353471.10002 - + @@ -752,6 +709,13 @@ + + + + + + + Index: src/contrib/hive/ql/src/test/results/compiler/plan/groupby5.q.xml =================================================================== --- src/contrib/hive/ql/src/test/results/compiler/plan/groupby5.q.xml (revision 711472) +++ src/contrib/hive/ql/src/test/results/compiler/plan/groupby5.q.xml (working copy) @@ -4,105 +4,7 @@ - - - - - - - Stage-1 - - - - - - - - - - - - - - - - - - - - - - true - - - /tmp/hive-zshao/218070299/172646370.10000.insclause-0 - - - - - org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe - - - org.apache.hadoop.mapred.TextInputFormat - - - org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat - - - - - name - dest1 - - - serialization.ddl - struct dest1 { string key, string value} - - - serialization.format - 1 - - - columns - key,value - - - bucket_count - -1 - - - serialization.lib - org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe - - - file.inputformat - org.apache.hadoop.mapred.TextInputFormat - - - file.outputformat - org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat - - - location - file:/data/users/zshao/tubbs-svnroot/projects/hadoop/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/dest1 - - - - - org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe - - - - - - - - - - - - - + Stage-3 @@ -118,7 +20,7 @@ - /tmp/hive-zshao/218070299/172646370.10001 + /tmp/hive-zshao/70834454/275752508.10002 @@ -144,25 +46,29 @@ - + - org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe + org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe - org.apache.hadoop.mapred.TextInputFormat + org.apache.hadoop.mapred.SequenceFileInputFormat - org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + org.apache.hadoop.mapred.SequenceFileOutputFormat - columns - 0 + name + binary_sortable_table + serialization.ddl + struct binary_sortable_table { string reducesinkkey0} + + serialization.format - 1 + org.apache.hadoop.hive.serde2.thrift.TBinarySortableProtocol @@ -185,32 +91,40 @@ 1 - + + + java.lang.Double + + - + - org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe + org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe - org.apache.hadoop.mapred.TextInputFormat + org.apache.hadoop.mapred.SequenceFileInputFormat - org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + org.apache.hadoop.mapred.SequenceFileOutputFormat - columns - 0 + name + binary_table + serialization.ddl + struct binary_table { double reducesinkvalue0} + + serialization.format - 1 + com.facebook.thrift.protocol.TBinaryProtocol @@ -238,7 +152,7 @@ VALUE.0 - + @@ -253,13 +167,16 @@ true + + + - /tmp/hive-zshao/218070299/172646370.10001 + /tmp/hive-zshao/70834454/275752508.10002 - /tmp/hive-zshao/218070299/172646370.10001 + /tmp/hive-zshao/70834454/275752508.10002 @@ -268,28 +185,32 @@ - /tmp/hive-zshao/218070299/172646370.10001 + /tmp/hive-zshao/70834454/275752508.10002 - + - org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe + org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe - org.apache.hadoop.mapred.TextInputFormat + org.apache.hadoop.mapred.SequenceFileInputFormat - org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + org.apache.hadoop.mapred.SequenceFileOutputFormat - columns - 0,1 + name + binary_table + serialization.ddl + struct binary_table { string temporarycol0, double temporarycol1} + + serialization.format - 1 + com.facebook.thrift.protocol.TBinaryProtocol @@ -312,10 +233,32 @@ - /tmp/hive-zshao/218070299/172646370.10000.insclause-0 + /tmp/hive-zshao/70834454/275752508.10001.insclause-0 - + + + org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe + + + org.apache.hadoop.mapred.TextInputFormat + + + org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + + + + + columns + key,_c1 + + + serialization.format + 1 + + + + @@ -339,7 +282,7 @@ 1 - + @@ -371,7 +314,7 @@ 1 - + @@ -407,7 +350,7 @@ VALUE.0 - + @@ -433,7 +376,7 @@ - PARTIAL2 + FINAL @@ -458,7 +401,7 @@ 1 - + @@ -468,6 +411,13 @@ + + + + + + + @@ -505,25 +455,29 @@ - + - org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe + org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe - org.apache.hadoop.mapred.TextInputFormat + org.apache.hadoop.mapred.SequenceFileInputFormat - org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + org.apache.hadoop.mapred.SequenceFileOutputFormat - columns - 0 + name + binary_sortable_table + serialization.ddl + struct binary_sortable_table { string reducesinkkey0} + + serialization.format - 1 + org.apache.hadoop.hive.serde2.thrift.TBinarySortableProtocol @@ -594,25 +548,29 @@ - + - org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe + org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe - org.apache.hadoop.mapred.TextInputFormat + org.apache.hadoop.mapred.SequenceFileInputFormat - org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + org.apache.hadoop.mapred.SequenceFileOutputFormat - columns - 0 + name + binary_table + serialization.ddl + struct binary_table { string reducesinkvalue0} + + serialization.format - 1 + com.facebook.thrift.protocol.TBinaryProtocol @@ -684,6 +642,9 @@ + + + @@ -773,10 +734,10 @@ - /tmp/hive-zshao/218070299/172646370.10001 + /tmp/hive-zshao/70834454/275752508.10002 - + @@ -800,7 +761,7 @@ 1 - + @@ -824,12 +785,36 @@ - - - VALUE.0 + + + org.apache.hadoop.hive.ql.udf.UDFToDouble + + + evaluate + + + java.lang.String + + + + + + + + + + VALUE.0 + + + + + + + + - + @@ -865,6 +850,13 @@ + + + + + + + Index: src/contrib/hive/ql/src/test/results/compiler/plan/groupby6.q.xml =================================================================== --- src/contrib/hive/ql/src/test/results/compiler/plan/groupby6.q.xml (revision 711472) +++ src/contrib/hive/ql/src/test/results/compiler/plan/groupby6.q.xml (working copy) @@ -4,105 +4,7 @@ - - - - - - - Stage-1 - - - - - - - - - - - - - - - - - - - - - - true - - - /tmp/hive-zshao/20627718/64699543.10000.insclause-0 - - - - - org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe - - - org.apache.hadoop.mapred.TextInputFormat - - - org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat - - - - - name - dest1 - - - serialization.ddl - struct dest1 { string key, string value} - - - serialization.format - 1 - - - columns - key,value - - - bucket_count - -1 - - - serialization.lib - org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe - - - file.inputformat - org.apache.hadoop.mapred.TextInputFormat - - - file.outputformat - org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat - - - location - file:/data/users/zshao/tubbs-svnroot/projects/hadoop/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/dest1 - - - - - org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe - - - - - - - - - - - - - + Stage-3 @@ -118,7 +20,7 @@ - /tmp/hive-zshao/20627718/64699543.10001 + /tmp/hive-zshao/1018494183/23473780.10002 @@ -144,25 +46,29 @@ - + - org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe + org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe - org.apache.hadoop.mapred.TextInputFormat + org.apache.hadoop.mapred.SequenceFileInputFormat - org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + org.apache.hadoop.mapred.SequenceFileOutputFormat - columns - 0 + name + binary_sortable_table + serialization.ddl + struct binary_sortable_table { string reducesinkkey0} + + serialization.format - 1 + org.apache.hadoop.hive.serde2.thrift.TBinarySortableProtocol @@ -181,25 +87,29 @@ - + - org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe + org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe - org.apache.hadoop.mapred.TextInputFormat + org.apache.hadoop.mapred.SequenceFileInputFormat - org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + org.apache.hadoop.mapred.SequenceFileOutputFormat - columns - + name + binary_table + serialization.ddl + struct binary_table { } + + serialization.format - 1 + com.facebook.thrift.protocol.TBinaryProtocol @@ -232,13 +142,16 @@ true + + + - /tmp/hive-zshao/20627718/64699543.10001 + /tmp/hive-zshao/1018494183/23473780.10002 - /tmp/hive-zshao/20627718/64699543.10001 + /tmp/hive-zshao/1018494183/23473780.10002 @@ -247,28 +160,32 @@ - /tmp/hive-zshao/20627718/64699543.10001 + /tmp/hive-zshao/1018494183/23473780.10002 - + - org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe + org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe - org.apache.hadoop.mapred.TextInputFormat + org.apache.hadoop.mapred.SequenceFileInputFormat - org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + org.apache.hadoop.mapred.SequenceFileOutputFormat - columns - 0 + name + binary_table + serialization.ddl + struct binary_table { string temporarycol0} + + serialization.format - 1 + com.facebook.thrift.protocol.TBinaryProtocol @@ -291,10 +208,32 @@ - /tmp/hive-zshao/20627718/64699543.10000.insclause-0 + /tmp/hive-zshao/1018494183/23473780.10001.insclause-0 - + + + org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe + + + org.apache.hadoop.mapred.TextInputFormat + + + org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + + + + + columns + _c0 + + + serialization.format + 1 + + + + @@ -370,7 +309,7 @@ - PARTIAL2 + FINAL @@ -395,6 +334,13 @@ + + + + + + + @@ -486,25 +432,29 @@ - + - org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe + org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe - org.apache.hadoop.mapred.TextInputFormat + org.apache.hadoop.mapred.SequenceFileInputFormat - org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + org.apache.hadoop.mapred.SequenceFileOutputFormat - columns - 0 + name + binary_sortable_table + serialization.ddl + struct binary_sortable_table { string reducesinkkey0} + + serialization.format - 1 + org.apache.hadoop.hive.serde2.thrift.TBinarySortableProtocol @@ -523,25 +473,29 @@ - + - org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe + org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe - org.apache.hadoop.mapred.TextInputFormat + org.apache.hadoop.mapred.SequenceFileInputFormat - org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + org.apache.hadoop.mapred.SequenceFileOutputFormat - columns - + name + binary_table + serialization.ddl + struct binary_table { } + + serialization.format - 1 + com.facebook.thrift.protocol.TBinaryProtocol @@ -603,6 +557,9 @@ + + + @@ -692,10 +649,10 @@ - /tmp/hive-zshao/20627718/64699543.10001 + /tmp/hive-zshao/1018494183/23473780.10002 - + @@ -752,6 +709,13 @@ + + + + + + + Index: src/contrib/hive/ql/src/test/results/compiler/plan/case_sensitivity.q.xml =================================================================== --- src/contrib/hive/ql/src/test/results/compiler/plan/case_sensitivity.q.xml (revision 711472) +++ src/contrib/hive/ql/src/test/results/compiler/plan/case_sensitivity.q.xml (working copy) @@ -31,7 +31,7 @@ true - /tmp/hive-zshao/111275125/273788462.10000.insclause-0 + /tmp/hive-zshao/130140254/281222924.10000.insclause-0 @@ -124,7 +124,7 @@ - /tmp/hive-zshao/111275125/273788462.10000.insclause-0 + /tmp/hive-zshao/130140254/281222924.10000.insclause-0 @@ -515,6 +515,9 @@ + + + Index: src/contrib/hive/ql/src/test/results/compiler/plan/sample1.q.xml =================================================================== --- src/contrib/hive/ql/src/test/results/compiler/plan/sample1.q.xml (revision 711472) +++ src/contrib/hive/ql/src/test/results/compiler/plan/sample1.q.xml (working copy) @@ -1,104 +1,6 @@ - - - - - - - Stage-1 - - - - - - - - - - - - - - - - - - - - - - true - - - /tmp/hive-zshao/130068324/266130293.10000.insclause-0 - - - - - org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe - - - org.apache.hadoop.mapred.TextInputFormat - - - org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat - - - - - name - dest1 - - - serialization.ddl - struct dest1 { string key, string value} - - - serialization.format - 1 - - - columns - key,value - - - bucket_count - -1 - - - serialization.lib - org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe - - - file.inputformat - org.apache.hadoop.mapred.TextInputFormat - - - file.outputformat - org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat - - - location - file:/data/users/zshao/tubbs-svnroot/projects/hadoop/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/dest1 - - - - - org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe - - - - - - - - - - - - - + Stage-2 @@ -116,65 +18,143 @@ - + - - - - - /tmp/hive-zshao/130068324/266130293.10000.insclause-0 + + + + + + + + + /tmp/hive-athusoo/15600693/557222875.10001.insclause-0 + + + + + org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe + + + org.apache.hadoop.mapred.TextInputFormat + + + org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + + + + + columns + key,value,ds,hr + + + serialization.format + 1 + + + + + + + + + + + + + + + 0 + + + + + java.lang.String + + + + + + + + + 1 + + + + + + + + + + 2 + + + + + + + + + + 3 + + + + + + + + + + + - - - - - - - + + + + - - - 0 + + + key - - - - java.lang.String - - + + - - - 1 + + + value - + - - - 2 + + + ds - + - - - 3 + + + hr - + @@ -183,50 +163,186 @@ + + + + + + + - - - + + + + + org.apache.hadoop.hive.ql.udf.UDFOPAnd + + + + evaluate + + + java.lang.Boolean + + + java.lang.Boolean + + + + + + + + + + org.apache.hadoop.hive.ql.udf.UDFOPEqual + + + + evaluate + + + java.lang.String + + + java.lang.String + + + + + + + + + + ds + + + + + + + + + + + + + 2008-04-08 + + + + + + + + + java.lang.Boolean + + + + + + + + + org.apache.hadoop.hive.ql.udf.UDFOPEqual + + + + evaluate + + + java.lang.String + + + java.lang.String + + + + + + + + + + hr + + + + + + + + + + + + + 11 + + + + + + + + + + + + + + + + + + + + + + + - - + + key - + - - + + value - + - - + + ds - + - - + + hr - + @@ -235,13 +351,6 @@ - - - - - - - @@ -251,17 +360,17 @@ - org.apache.hadoop.hive.ql.udf.UDFOPAnd + org.apache.hadoop.hive.ql.udf.UDFOPEqual - + evaluate - java.lang.Boolean + java.lang.Integer - java.lang.Boolean + java.lang.Integer @@ -271,17 +380,17 @@ - org.apache.hadoop.hive.ql.udf.UDFOPEqual + org.apache.hadoop.hive.ql.udf.UDFOPMod - + evaluate - java.lang.String + java.lang.Integer - java.lang.String + java.lang.Integer @@ -289,81 +398,118 @@ - - - ds + + + org.apache.hadoop.hive.ql.udf.UDFOPBitAnd + + + evaluate + + + java.lang.Integer + + + java.lang.Integer + + + + + + + + + + org.apache.hadoop.hive.ql.udf.UDFDefaultSampleHashFn + + + + evaluate + + + java.lang.Object + + + + + + + + + + org.apache.hadoop.hive.ql.udf.UDFRand + + + + evaluate + + + + + + + + + + java.lang.Double + + + + + + + + + + + java.lang.Integer + + + + + + + + + + + + 2147483647 + + + + + - + - + - 2008-04-08 + 1 - - - java.lang.Boolean - - + - - - org.apache.hadoop.hive.ql.udf.UDFOPEqual + + + - - - evaluate - - - java.lang.String - - - java.lang.String - - - + + 0 - - - - - - hr - - - - - - - - - - - - - 11 - - - - - - - - @@ -375,54 +521,6 @@ - - - - - - - - key - - - - - - - - - - value - - - - - - - - - - ds - - - - - - - - - - hr - - - - - - - - - - @@ -441,7 +539,7 @@ - file:/data/users/zshao/tubbs-svnroot/projects/hadoop/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/srcpart/ds=2008-04-08/hr=11 + file:/data/users/athusoo/hadoop_local_ws1/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/srcpart/ds=2008-04-08/hr=11 s @@ -453,7 +551,7 @@ - file:/data/users/zshao/tubbs-svnroot/projects/hadoop/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/srcpart/ds=2008-04-08/hr=11 + file:/data/users/athusoo/hadoop_local_ws1/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/srcpart/ds=2008-04-08/hr=11 @@ -518,7 +616,7 @@ location - file:/data/users/zshao/tubbs-svnroot/projects/hadoop/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/srcpart + file:/data/users/athusoo/hadoop_local_ws1/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/srcpart @@ -531,6 +629,9 @@ + + + Index: src/contrib/hive/ql/src/test/results/compiler/plan/sample2.q.xml =================================================================== --- src/contrib/hive/ql/src/test/results/compiler/plan/sample2.q.xml (revision 711472) +++ src/contrib/hive/ql/src/test/results/compiler/plan/sample2.q.xml (working copy) @@ -31,7 +31,7 @@ true - /tmp/hive-zshao/741118865/96139643.10000.insclause-0 + /tmp/hive-athusoo/19832992/267616877.10000.insclause-0 @@ -80,7 +80,7 @@ location - file:/data/users/zshao/tubbs-svnroot/projects/hadoop/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/dest1 + file:/data/users/athusoo/hadoop_local_ws1/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/dest1 @@ -120,7 +120,7 @@ - /tmp/hive-zshao/741118865/96139643.10000.insclause-0 + /tmp/hive-athusoo/19832992/267616877.10000.insclause-0 @@ -237,7 +237,7 @@ - file:/data/users/zshao/tubbs-svnroot/projects/hadoop/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/srcbucket/kv1.txt + file:/data/users/athusoo/hadoop_local_ws1/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/srcbucket/kv1.txt s @@ -249,7 +249,7 @@ - file:/data/users/zshao/tubbs-svnroot/projects/hadoop/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/srcbucket/kv1.txt + file:/data/users/athusoo/hadoop_local_ws1/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/srcbucket/kv1.txt @@ -272,18 +272,22 @@ srcbucket + bucket_field_name + key + + serialization.ddl struct srcbucket { string key, string value} + columns + key,value + + serialization.format 1 - columns - key,value - - bucket_count 2 @@ -301,7 +305,7 @@ location - file:/data/users/zshao/tubbs-svnroot/projects/hadoop/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/srcbucket + file:/data/users/athusoo/hadoop_local_ws1/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/srcbucket @@ -314,6 +318,9 @@ + + + Index: src/contrib/hive/ql/src/test/results/compiler/plan/sample3.q.xml =================================================================== --- src/contrib/hive/ql/src/test/results/compiler/plan/sample3.q.xml (revision 711472) +++ src/contrib/hive/ql/src/test/results/compiler/plan/sample3.q.xml (working copy) @@ -31,7 +31,7 @@ true - /tmp/hive-zshao/1895709606/245717296.10000.insclause-0 + /tmp/hive-athusoo/586773862/1314113986.10000.insclause-0 @@ -80,7 +80,7 @@ location - file:/data/users/zshao/tubbs-svnroot/projects/hadoop/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/dest1 + file:/data/users/athusoo/hadoop_local_ws1/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/dest1 @@ -124,7 +124,7 @@ - /tmp/hive-zshao/1895709606/245717296.10000.insclause-0 + /tmp/hive-athusoo/586773862/1314113986.10000.insclause-0 @@ -415,7 +415,7 @@ - file:/data/users/zshao/tubbs-svnroot/projects/hadoop/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/srcbucket + file:/data/users/athusoo/hadoop_local_ws1/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/srcbucket s @@ -427,7 +427,7 @@ - file:/data/users/zshao/tubbs-svnroot/projects/hadoop/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/srcbucket + file:/data/users/athusoo/hadoop_local_ws1/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/srcbucket @@ -450,18 +450,22 @@ srcbucket + bucket_field_name + key + + serialization.ddl struct srcbucket { string key, string value} + columns + key,value + + serialization.format 1 - columns - key,value - - bucket_count 2 @@ -479,7 +483,7 @@ location - file:/data/users/zshao/tubbs-svnroot/projects/hadoop/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/srcbucket + file:/data/users/athusoo/hadoop_local_ws1/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/srcbucket @@ -492,6 +496,9 @@ + + + Index: src/contrib/hive/ql/src/test/results/compiler/plan/sample4.q.xml =================================================================== --- src/contrib/hive/ql/src/test/results/compiler/plan/sample4.q.xml (revision 711472) +++ src/contrib/hive/ql/src/test/results/compiler/plan/sample4.q.xml (working copy) @@ -31,7 +31,7 @@ true - /tmp/hive-zshao/614066513/551990576.10000.insclause-0 + /tmp/hive-athusoo/2564982/226525296.10000.insclause-0 @@ -80,7 +80,7 @@ location - file:/data/users/zshao/tubbs-svnroot/projects/hadoop/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/dest1 + file:/data/users/athusoo/hadoop_local_ws1/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/dest1 @@ -112,81 +112,45 @@ - + - - - - - - - - - /tmp/hive-zshao/614066513/551990576.10000.insclause-0 - - - - - - - - - - - - - - 0 - - - - - java.lang.String - - - - - - - - - 1 - - - - - - - - - - - + + + + + /tmp/hive-athusoo/2564982/226525296.10000.insclause-0 + + + - - - - + + + + - - - key + + + 0 - - + + + + java.lang.String + + - - - value + + + 1 - + @@ -195,174 +159,45 @@ - - - - - - - - - - - - org.apache.hadoop.hive.ql.udf.UDFOPEqual - - - - evaluate - - - java.lang.Integer - - - java.lang.Integer - - - - - - - - - - org.apache.hadoop.hive.ql.udf.UDFOPMod - - - - evaluate - - - java.lang.Integer - - - java.lang.Integer - - - - - - - - - - org.apache.hadoop.hive.ql.udf.UDFOPBitAnd - - - - evaluate - - - java.lang.Integer - - - java.lang.Integer - - - - - - - - - - org.apache.hadoop.hive.ql.udf.UDFDefaultSampleHashFn - - - - evaluate - - - java.lang.Object - - - - - - - - - - key - - - - - - - - - - - - java.lang.Integer - - - - - - - - - - - - 2147483647 - - - - - - - - - - - - - - - - - 2 - - - - - - - - - + + + + + + + key - - - - - - - 0 - - + + - - - - java.lang.Boolean + + + + value + + + + + + + + + + @@ -402,7 +237,7 @@ - file:/data/users/zshao/tubbs-svnroot/projects/hadoop/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/srcbucket + file:/data/users/athusoo/hadoop_local_ws1/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/srcbucket/kv1.txt s @@ -414,7 +249,7 @@ - file:/data/users/zshao/tubbs-svnroot/projects/hadoop/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/srcbucket + file:/data/users/athusoo/hadoop_local_ws1/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/srcbucket/kv1.txt @@ -437,18 +272,22 @@ srcbucket + bucket_field_name + key + + serialization.ddl struct srcbucket { string key, string value} - serialization.format - 1 - - columns key,value + serialization.format + 1 + + bucket_count 2 @@ -466,7 +305,7 @@ location - file:/data/users/zshao/tubbs-svnroot/projects/hadoop/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/srcbucket + file:/data/users/athusoo/hadoop_local_ws1/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/srcbucket @@ -479,6 +318,9 @@ + + + Index: src/contrib/hive/ql/src/test/results/compiler/plan/sample5.q.xml =================================================================== --- src/contrib/hive/ql/src/test/results/compiler/plan/sample5.q.xml (revision 711472) +++ src/contrib/hive/ql/src/test/results/compiler/plan/sample5.q.xml (working copy) @@ -31,7 +31,7 @@ true - /tmp/hive-zshao/532954132/21366373.10000.insclause-0 + /tmp/hive-athusoo/141731932/482635954.10000.insclause-0 @@ -80,7 +80,7 @@ location - file:/data/users/zshao/tubbs-svnroot/projects/hadoop/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/dest1 + file:/data/users/athusoo/hadoop_local_ws1/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/dest1 @@ -124,7 +124,7 @@ - /tmp/hive-zshao/532954132/21366373.10000.insclause-0 + /tmp/hive-athusoo/141731932/482635954.10000.insclause-0 @@ -402,7 +402,7 @@ - file:/data/users/zshao/tubbs-svnroot/projects/hadoop/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/srcbucket + file:/data/users/athusoo/hadoop_local_ws1/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/srcbucket s @@ -414,7 +414,7 @@ - file:/data/users/zshao/tubbs-svnroot/projects/hadoop/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/srcbucket + file:/data/users/athusoo/hadoop_local_ws1/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/srcbucket @@ -437,18 +437,22 @@ srcbucket + bucket_field_name + key + + serialization.ddl struct srcbucket { string key, string value} + columns + key,value + + serialization.format 1 - columns - key,value - - bucket_count 2 @@ -466,7 +470,7 @@ location - file:/data/users/zshao/tubbs-svnroot/projects/hadoop/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/srcbucket + file:/data/users/athusoo/hadoop_local_ws1/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/srcbucket @@ -479,6 +483,9 @@ + + + Index: src/contrib/hive/ql/src/test/results/compiler/plan/sample6.q.xml =================================================================== --- src/contrib/hive/ql/src/test/results/compiler/plan/sample6.q.xml (revision 711472) +++ src/contrib/hive/ql/src/test/results/compiler/plan/sample6.q.xml (working copy) @@ -31,7 +31,7 @@ true - /tmp/hive-zshao/554218395/580685485.10000.insclause-0 + /tmp/hive-athusoo/211424568/185314938.10000.insclause-0 @@ -80,7 +80,7 @@ location - file:/data/users/zshao/tubbs-svnroot/projects/hadoop/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/dest1 + file:/data/users/athusoo/hadoop_local_ws1/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/dest1 @@ -124,7 +124,7 @@ - /tmp/hive-zshao/554218395/580685485.10000.insclause-0 + /tmp/hive-athusoo/211424568/185314938.10000.insclause-0 @@ -402,7 +402,7 @@ - file:/data/users/zshao/tubbs-svnroot/projects/hadoop/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/srcbucket + file:/data/users/athusoo/hadoop_local_ws1/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/srcbucket/kv1.txt s @@ -414,7 +414,7 @@ - file:/data/users/zshao/tubbs-svnroot/projects/hadoop/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/srcbucket + file:/data/users/athusoo/hadoop_local_ws1/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/srcbucket/kv1.txt @@ -437,18 +437,22 @@ srcbucket + bucket_field_name + key + + serialization.ddl struct srcbucket { string key, string value} + columns + key,value + + serialization.format 1 - columns - key,value - - bucket_count 2 @@ -466,7 +470,7 @@ location - file:/data/users/zshao/tubbs-svnroot/projects/hadoop/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/srcbucket + file:/data/users/athusoo/hadoop_local_ws1/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/srcbucket @@ -479,6 +483,9 @@ + + + Index: src/contrib/hive/ql/src/test/results/compiler/plan/sample7.q.xml =================================================================== --- src/contrib/hive/ql/src/test/results/compiler/plan/sample7.q.xml (revision 711472) +++ src/contrib/hive/ql/src/test/results/compiler/plan/sample7.q.xml (working copy) @@ -31,7 +31,7 @@ true - /tmp/hive-zshao/393286311/133813526.10000.insclause-0 + /tmp/hive-athusoo/266475143/404065996.10000.insclause-0 @@ -80,7 +80,7 @@ location - file:/data/users/zshao/tubbs-svnroot/projects/hadoop/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/dest1 + file:/data/users/athusoo/hadoop_local_ws1/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/dest1 @@ -128,7 +128,7 @@ - /tmp/hive-zshao/393286311/133813526.10000.insclause-0 + /tmp/hive-athusoo/266475143/404065996.10000.insclause-0 @@ -468,7 +468,7 @@ - file:/data/users/zshao/tubbs-svnroot/projects/hadoop/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/srcbucket + file:/data/users/athusoo/hadoop_local_ws1/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/srcbucket/kv1.txt s @@ -480,7 +480,7 @@ - file:/data/users/zshao/tubbs-svnroot/projects/hadoop/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/srcbucket + file:/data/users/athusoo/hadoop_local_ws1/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/srcbucket/kv1.txt @@ -503,18 +503,22 @@ srcbucket + bucket_field_name + key + + serialization.ddl struct srcbucket { string key, string value} + columns + key,value + + serialization.format 1 - columns - key,value - - bucket_count 2 @@ -532,7 +536,7 @@ location - file:/data/users/zshao/tubbs-svnroot/projects/hadoop/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/srcbucket + file:/data/users/athusoo/hadoop_local_ws1/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/srcbucket @@ -545,6 +549,9 @@ + + + Index: src/contrib/hive/ql/src/test/results/compiler/plan/cast1.q.xml =================================================================== --- src/contrib/hive/ql/src/test/results/compiler/plan/cast1.q.xml (revision 711472) +++ src/contrib/hive/ql/src/test/results/compiler/plan/cast1.q.xml (working copy) @@ -1,104 +1,6 @@ - - - - - - - Stage-1 - - - - - - - - - - - - - - - - - - - - - - true - - - /tmp/hive-zshao/783071627/588980298.10000.insclause-0 - - - - - org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe - - - org.apache.hadoop.mapred.TextInputFormat - - - org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat - - - - - name - dest1 - - - serialization.ddl - struct dest1 { string key, string value} - - - serialization.format - 1 - - - columns - key,value - - - bucket_count - -1 - - - serialization.lib - org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe - - - file.inputformat - org.apache.hadoop.mapred.TextInputFormat - - - file.outputformat - org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat - - - location - file:/data/users/zshao/tubbs-svnroot/projects/hadoop/trunk/VENDOR/hadoop-0.17/build/contrib/hive/ql/test/data/warehouse/dest1 - - - - - org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe - - - - - - - - - - - - - + Stage-2 @@ -124,10 +26,32 @@ - /tmp/hive-zshao/783071627/588980298.10000.insclause-0 + /tmp/hive-zshao/43367676/365255705.10001.insclause-0 - + + + org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe + + + org.apache.hadoop.mapred.TextInputFormat + + + org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + + + + + columns + _c0,_c1,_c2,_c3,_c4,_c5,_c6 + + + serialization.format + 1 + + + + @@ -797,6 +721,9 @@ + + + Index: src/contrib/hive/ql/src/test/results/compiler/plan/join1.q.xml =================================================================== --- src/contrib/hive/ql/src/test/results/compiler/plan/join1.q.xml (revision 711472) +++ src/contrib/hive/ql/src/test/results/compiler/plan/join1.q.xml (working copy) @@ -31,7 +31,7 @@ true - /tmp/hive-zshao/108001173/349936017.10000.insclause-0 + /tmp/hive-zshao/936281633/614726067.10000.insclause-0 @@ -136,23 +136,27 @@ - org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe + org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe - org.apache.hadoop.mapred.TextInputFormat + org.apache.hadoop.mapred.SequenceFileInputFormat - org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + org.apache.hadoop.mapred.SequenceFileOutputFormat - columns - 0 + name + binary_sortable_table + serialization.ddl + struct binary_sortable_table { string joinkey0} + + serialization.format - 1 + org.apache.hadoop.hive.serde2.thrift.TBinarySortableProtocol @@ -192,25 +196,29 @@ - + - org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe + org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe - org.apache.hadoop.mapred.TextInputFormat + org.apache.hadoop.mapred.SequenceFileInputFormat - org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + org.apache.hadoop.mapred.SequenceFileOutputFormat - columns - 0,1 + name + binary_table + serialization.ddl + struct binary_table { string reducesinkvalue0, string reducesinkvalue1} + + serialization.format - 1 + com.facebook.thrift.protocol.TBinaryProtocol @@ -304,25 +312,29 @@ - + - org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe + org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe - org.apache.hadoop.mapred.TextInputFormat + org.apache.hadoop.mapred.SequenceFileInputFormat - org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + org.apache.hadoop.mapred.SequenceFileOutputFormat - columns - 0 + name + binary_sortable_table + serialization.ddl + struct binary_sortable_table { string joinkey0} + + serialization.format - 1 + org.apache.hadoop.hive.serde2.thrift.TBinarySortableProtocol @@ -359,25 +371,29 @@ - + - org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe + org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe - org.apache.hadoop.mapred.TextInputFormat + org.apache.hadoop.mapred.SequenceFileInputFormat - org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + org.apache.hadoop.mapred.SequenceFileOutputFormat - columns - 0,1 + name + binary_table + serialization.ddl + struct binary_table { string reducesinkvalue0, string reducesinkvalue1} + + serialization.format - 1 + com.facebook.thrift.protocol.TBinaryProtocol @@ -449,6 +465,9 @@ + + + true @@ -548,7 +567,7 @@ - /tmp/hive-zshao/108001173/349936017.10000.insclause-0 + /tmp/hive-zshao/936281633/614726067.10000.insclause-0 @@ -745,6 +764,16 @@ + + + + + + + + + + Index: src/contrib/hive/ql/src/test/results/compiler/plan/input1.q.xml =================================================================== --- src/contrib/hive/ql/src/test/results/compiler/plan/input1.q.xml (revision 711472) +++ src/contrib/hive/ql/src/test/results/compiler/plan/input1.q.xml (working copy) @@ -31,7 +31,7 @@ true - /tmp/hive-zshao/374447248/1282977307.10000.insclause-0 + /tmp/hive-zshao/156066098/28125611.10000.insclause-0 @@ -124,7 +124,7 @@ - /tmp/hive-zshao/374447248/1282977307.10000.insclause-0 + /tmp/hive-zshao/156066098/28125611.10000.insclause-0 @@ -388,6 +388,9 @@ + + + Index: src/contrib/hive/ql/src/test/results/compiler/errors/notable_alias1.q.out =================================================================== --- src/contrib/hive/ql/src/test/results/compiler/errors/notable_alias1.q.out (revision 711472) +++ src/contrib/hive/ql/src/test/results/compiler/errors/notable_alias1.q.out (working copy) @@ -1 +0,0 @@ -Parse Error: line 2:44 cannot recognize input 'value' Index: src/contrib/hive/ql/src/test/results/compiler/errors/notable_alias2.q.out =================================================================== --- src/contrib/hive/ql/src/test/results/compiler/errors/notable_alias2.q.out (revision 711472) +++ src/contrib/hive/ql/src/test/results/compiler/errors/notable_alias2.q.out (working copy) @@ -1 +0,0 @@ -Parse Error: line 2:60 cannot recognize input 'key' Index: src/contrib/hive/ql/src/test/results/compiler/errors/notable_alias3.q.out =================================================================== --- src/contrib/hive/ql/src/test/results/compiler/errors/notable_alias3.q.out (revision 711472) +++ src/contrib/hive/ql/src/test/results/compiler/errors/notable_alias3.q.out (working copy) @@ -1 +0,0 @@ -Parse Error: line 2:83 cannot recognize input 'key' Index: src/contrib/hive/ql/src/test/results/compiler/errors/quoted_string.q.out =================================================================== --- src/contrib/hive/ql/src/test/results/compiler/errors/quoted_string.q.out (revision 711472) +++ src/contrib/hive/ql/src/test/results/compiler/errors/quoted_string.q.out (working copy) @@ -1,2 +1,2 @@ -Parse Error: line 2:36 character '"' not supported here -line 2:41 character '"' not supported here +Parse Error: line 3:0 character '' not supported here +line 3:0 character '' not supported here Index: src/contrib/hive/ql/src/test/results/compiler/errors/insert_wrong_number_columns.q.out =================================================================== --- src/contrib/hive/ql/src/test/results/compiler/errors/insert_wrong_number_columns.q.out (revision 0) +++ src/contrib/hive/ql/src/test/results/compiler/errors/insert_wrong_number_columns.q.out (revision 0) @@ -0,0 +1,2 @@ +Semantic Exception: +line 2:23 Cannot insert into target table because column number/types are different dest1: Table insclause-0 has 2 columns but query has [0: string, 1: string, 2: int]. \ No newline at end of file Index: src/contrib/hive/ql/src/test/org/apache/hadoop/hive/ql/exec/TestExecDriver.java =================================================================== --- src/contrib/hive/ql/src/test/org/apache/hadoop/hive/ql/exec/TestExecDriver.java (revision 711472) +++ src/contrib/hive/ql/src/test/org/apache/hadoop/hive/ql/exec/TestExecDriver.java (working copy) @@ -198,7 +198,7 @@ @SuppressWarnings("unchecked") private void populateMapRedPlan1(Table src) { mr.setNumReduceTasks(Integer.valueOf(1)); - + // map-side work Operator op1 = OperatorFactory.get (PlanUtils.getReduceSinkDesc @@ -206,6 +206,8 @@ Utilities.makeList(new exprNodeColumnDesc(String.class, "value")), -1, 1, -1, false)); Utilities.addMapWork(mr, src, "a", op1); + mr.setKeyDesc(op1.getConf().getKeySerializeInfo()); + mr.getTagToValueDesc().add(op1.getConf().getValueSerializeInfo()); // reduce side work Operator op3 = OperatorFactory.get(new fileSinkDesc @@ -230,6 +232,8 @@ new exprNodeColumnDesc(String.class, "value")), -1, 1, -1, false)); Utilities.addMapWork(mr, src, "a", op1); + mr.setKeyDesc(op1.getConf().getKeySerializeInfo()); + mr.getTagToValueDesc().add(op1.getConf().getValueSerializeInfo()); // reduce side work Operator op4 = OperatorFactory.get(new fileSinkDesc @@ -261,6 +265,8 @@ (new exprNodeColumnDesc(String.class, "value")), Byte.valueOf((byte)0), 1, -1, false)); Utilities.addMapWork(mr, src, "a", op1); + mr.setKeyDesc(op1.getConf().getKeySerializeInfo()); + mr.getTagToValueDesc().add(op1.getConf().getValueSerializeInfo()); Operator op2 = OperatorFactory.get (PlanUtils.getReduceSinkDesc @@ -270,11 +276,8 @@ Integer.MAX_VALUE, -1, false)); Utilities.addMapWork(mr, src2, "b", op2); + mr.getTagToValueDesc().add(op2.getConf().getValueSerializeInfo()); - // just to satisfy the constraint that each tag must define a schema - mr.getAliasToSchema().put("a", new schemaDesc("")); - mr.getAliasToSchema().put("b", new schemaDesc("")); - // reduce side work Operator op4 = OperatorFactory.get(new fileSinkDesc (tmpdir + "mapredplan3.out", @@ -318,6 +321,8 @@ new exprNodeColumnDesc(String.class, "value"))), op0); Utilities.addMapWork(mr, src, "a", op4); + mr.setKeyDesc(op1.getConf().getKeySerializeInfo()); + mr.getTagToValueDesc().add(op1.getConf().getValueSerializeInfo()); // reduce side work Operator op3 = OperatorFactory.get(new fileSinkDesc @@ -348,6 +353,8 @@ new exprNodeColumnDesc(String.class, "value"))), op0); Utilities.addMapWork(mr, src, "a", op4); + mr.setKeyDesc(op0.getConf().getKeySerializeInfo()); + mr.getTagToValueDesc().add(op0.getConf().getValueSerializeInfo()); // reduce side work Operator op3 = OperatorFactory.get(new fileSinkDesc @@ -384,6 +391,8 @@ new exprNodeColumnDesc(String.class, "value"))), op0); Utilities.addMapWork(mr, src, "a", op4); + mr.setKeyDesc(op1.getConf().getKeySerializeInfo()); + mr.getTagToValueDesc().add(op1.getConf().getValueSerializeInfo()); // reduce side work Operator op3 = OperatorFactory.get(new fileSinkDesc Index: src/contrib/hive/ql/src/test/scripts/testgrep =================================================================== --- src/contrib/hive/ql/src/test/scripts/testgrep (revision 0) +++ src/contrib/hive/ql/src/test/scripts/testgrep (revision 0) @@ -0,0 +1,5 @@ +#!/bin/bash + +egrep '10.*' + +exit 0; Property changes on: src/contrib/hive/ql/src/test/scripts/testgrep ___________________________________________________________________ Added: svn:executable + * Index: src/contrib/hive/ql/src/test/queries/clientnegative/notable_alias3.q =================================================================== --- src/contrib/hive/ql/src/test/queries/clientnegative/notable_alias3.q (revision 0) +++ src/contrib/hive/ql/src/test/queries/clientnegative/notable_alias3.q (revision 0) @@ -0,0 +1,4 @@ +CREATE TABLE dest1(key INT, value DOUBLE) STORED AS TEXTFILE; + +FROM src +INSERT OVERWRITE TABLE dest1 SELECT '1234', src.key, sum(src.value) WHERE src.key < 100 group by key; Index: src/contrib/hive/ql/src/test/queries/clientnegative/notable_alias4.q =================================================================== --- src/contrib/hive/ql/src/test/queries/clientnegative/notable_alias4.q (revision 0) +++ src/contrib/hive/ql/src/test/queries/clientnegative/notable_alias4.q (revision 0) @@ -0,0 +1,4 @@ +EXPLAIN +SELECT key from src JOIN src1 on src1.key=src.key; + +SELECT key from src JOIN src1 on src1.key=src.key; Index: src/contrib/hive/ql/src/test/queries/clientnegative/bad_sample_clause.q =================================================================== --- src/contrib/hive/ql/src/test/queries/clientnegative/bad_sample_clause.q (revision 0) +++ src/contrib/hive/ql/src/test/queries/clientnegative/bad_sample_clause.q (revision 0) @@ -0,0 +1,6 @@ +CREATE TABLE dest1(key INT, value STRING, dt STRING, hr STRING) STORED AS TEXTFILE; + +INSERT OVERWRITE TABLE dest1 SELECT s.* +FROM srcpart TABLESAMPLE (BUCKET 1 OUT OF 2) s +WHERE s.ds='2008-04-08' and s.hr='11'; + Index: src/contrib/hive/ql/src/test/queries/clientnegative/load_wrong_fileformat.q =================================================================== --- src/contrib/hive/ql/src/test/queries/clientnegative/load_wrong_fileformat.q (revision 0) +++ src/contrib/hive/ql/src/test/queries/clientnegative/load_wrong_fileformat.q (revision 0) @@ -0,0 +1,6 @@ +-- test for loading into tables with the correct file format +-- test for loading into partitions with the correct file format + +DROP TABLE T1; +CREATE TABLE T1(name STRING) STORED AS SEQUENCEFILE; +LOAD DATA LOCAL INPATH '../data/files/kv1.txt' INTO TABLE T1; Index: src/contrib/hive/ql/src/test/queries/clientnegative/input_testxpath4.q =================================================================== --- src/contrib/hive/ql/src/test/queries/clientnegative/input_testxpath4.q (revision 0) +++ src/contrib/hive/ql/src/test/queries/clientnegative/input_testxpath4.q (revision 0) @@ -0,0 +1,6 @@ +EXPLAIN +FROM src_thrift +SELECT src_thrift.mstringstring['key_9'], lintstring.myint; + +FROM src_thrift +SELECT src_thrift.mstringstring['key_9'], lintstring.myint; Index: src/contrib/hive/ql/src/test/queries/clientnegative/invalid_tbl_name.q =================================================================== --- src/contrib/hive/ql/src/test/queries/clientnegative/invalid_tbl_name.q (revision 0) +++ src/contrib/hive/ql/src/test/queries/clientnegative/invalid_tbl_name.q (revision 0) @@ -0,0 +1 @@ +create table invalid-name(a int, b string); Index: src/contrib/hive/ql/src/test/queries/clientnegative/invalid_create_tbl1.q =================================================================== --- src/contrib/hive/ql/src/test/queries/clientnegative/invalid_create_tbl1.q (revision 0) +++ src/contrib/hive/ql/src/test/queries/clientnegative/invalid_create_tbl1.q (revision 0) @@ -0,0 +1,7 @@ +DROP TABLE inv_valid_tbl1; +CREATE TABLE inv_valid_tbl1 COMMENT 'This is a thrift based table' + PARTITIONED BY(aint DATETIME, country STRING) + CLUSTERED BY(aint) SORTED BY(lint) INTO 32 BUCKETS + ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.ThriftDeserializer' WITH SERDEPROPERTIES ('serialization.class' = 'org.apache.hadoop.hive.serde2.thrift.test.Complex', 'serialization.format' = 'com.facebook.thrift.protocol.TBinaryProtocol') + STORED AS SEQUENCEFILE; +DESCRIBE EXTENDED inv_valid_tbl1; Index: src/contrib/hive/ql/src/test/queries/clientpositive/alter1.q =================================================================== --- src/contrib/hive/ql/src/test/queries/clientpositive/alter1.q (revision 0) +++ src/contrib/hive/ql/src/test/queries/clientpositive/alter1.q (revision 0) @@ -0,0 +1,20 @@ +drop table alter1; +create table alter1(a int, b int); +describe extended alter1; +alter table alter1 set tblproperties ('a'='1', 'c'='3'); +describe extended alter1; +alter table alter1 set tblproperties ('a'='1', 'c'='4', 'd'='3'); +describe extended alter1; + +alter table alter1 set serdeproperties('s1'='9'); +describe extended alter1; +alter table alter1 set serdeproperties('s1'='10', 's2' ='20'); +describe extended alter1; + +alter table alter1 set serde 'org.apache.hadoop.hive.serde2.TestSerDe' with serdeproperties('s1'='9'); +describe extended alter1; + +alter table alter1 set serde 'org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe'; +describe extended alter1; + +drop table alter1; Index: src/contrib/hive/ql/src/test/queries/clientpositive/input11_limit.q =================================================================== --- src/contrib/hive/ql/src/test/queries/clientpositive/input11_limit.q (revision 711472) +++ src/contrib/hive/ql/src/test/queries/clientpositive/input11_limit.q (working copy) @@ -1,4 +1,4 @@ -CREATE TABLE dest1(key INT, value STRING); +CREATE TABLE dest1(key INT, value STRING) STORED AS TEXTFILE; EXPLAIN FROM src Index: src/contrib/hive/ql/src/test/queries/clientpositive/input14_limit.q =================================================================== --- src/contrib/hive/ql/src/test/queries/clientpositive/input14_limit.q (revision 711472) +++ src/contrib/hive/ql/src/test/queries/clientpositive/input14_limit.q (working copy) @@ -1,4 +1,4 @@ -CREATE TABLE dest1(key INT, value STRING); +CREATE TABLE dest1(key INT, value STRING) STORED AS TEXTFILE; EXPLAIN FROM ( Index: src/contrib/hive/ql/src/test/queries/clientpositive/sample2.q =================================================================== --- src/contrib/hive/ql/src/test/queries/clientpositive/sample2.q (revision 711472) +++ src/contrib/hive/ql/src/test/queries/clientpositive/sample2.q (working copy) @@ -1,4 +1,4 @@ -CREATE TABLE dest1(key INT, value STRING); +CREATE TABLE dest1(key INT, value STRING) STORED AS TEXTFILE; -- input pruning, no sample filter -- default table sample columns Index: src/contrib/hive/ql/src/test/queries/clientpositive/inputddl1.q =================================================================== --- src/contrib/hive/ql/src/test/queries/clientpositive/inputddl1.q (revision 711472) +++ src/contrib/hive/ql/src/test/queries/clientpositive/inputddl1.q (working copy) @@ -1,7 +1,7 @@ EXPLAIN -CREATE TABLE INPUTDDL1(key INT, value STRING); +CREATE TABLE INPUTDDL1(key INT, value STRING) STORED AS TEXTFILE; -CREATE TABLE INPUTDDL1(key INT, value STRING); +CREATE TABLE INPUTDDL1(key INT, value STRING) STORED AS TEXTFILE; SELECT INPUTDDL1.* from INPUTDDL1; Index: src/contrib/hive/ql/src/test/queries/clientpositive/sample4.q =================================================================== --- src/contrib/hive/ql/src/test/queries/clientpositive/sample4.q (revision 711472) +++ src/contrib/hive/ql/src/test/queries/clientpositive/sample4.q (working copy) @@ -1,4 +1,4 @@ -CREATE TABLE dest1(key INT, value STRING); +CREATE TABLE dest1(key INT, value STRING) STORED AS TEXTFILE; -- bucket column is the same as table sample -- No need for sample filter Index: src/contrib/hive/ql/src/test/queries/clientpositive/inputddl3.q =================================================================== --- src/contrib/hive/ql/src/test/queries/clientpositive/inputddl3.q (revision 711472) +++ src/contrib/hive/ql/src/test/queries/clientpositive/inputddl3.q (working copy) @@ -1,5 +1,5 @@ EXPLAIN -CREATE TABLE INPUTDDL3(key INT, value STRING) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t'; -CREATE TABLE INPUTDDL3(key INT, value STRING) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t'; +CREATE TABLE INPUTDDL3(key INT, value STRING) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t' STORED AS TEXTFILE; +CREATE TABLE INPUTDDL3(key INT, value STRING) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t' STORED AS TEXTFILE; DESCRIBE INPUTDDL3; DROP TABLE INPUTDDL3; Index: src/contrib/hive/ql/src/test/queries/clientpositive/groupby2_map.q =================================================================== --- src/contrib/hive/ql/src/test/queries/clientpositive/groupby2_map.q (revision 0) +++ src/contrib/hive/ql/src/test/queries/clientpositive/groupby2_map.q (revision 0) @@ -0,0 +1,12 @@ +set hive.map.aggr=true; + +CREATE TABLE dest1(key STRING, c1 INT, c2 STRING) STORED AS TEXTFILE; + +EXPLAIN +FROM src +INSERT OVERWRITE TABLE dest1 SELECT substr(src.key,0,1), count(DISTINCT substr(src.value,4)), concat(substr(src.key,0,1),sum(substr(src.value,4))) GROUP BY substr(src.key,0,1); + +FROM src +INSERT OVERWRITE TABLE dest1 SELECT substr(src.key,0,1), count(DISTINCT substr(src.value,4)), concat(substr(src.key,0,1),sum(substr(src.value,4))) GROUP BY substr(src.key,0,1); + +SELECT dest1.* FROM dest1; Index: src/contrib/hive/ql/src/test/queries/clientpositive/sample6.q =================================================================== --- src/contrib/hive/ql/src/test/queries/clientpositive/sample6.q (revision 711472) +++ src/contrib/hive/ql/src/test/queries/clientpositive/sample6.q (working copy) @@ -1,4 +1,4 @@ -CREATE TABLE dest1(key INT, value STRING); +CREATE TABLE dest1(key INT, value STRING) STORED AS TEXTFILE; -- both input pruning and sample filter EXPLAIN EXTENDED Index: src/contrib/hive/ql/src/test/queries/clientpositive/groupby4_map.q =================================================================== --- src/contrib/hive/ql/src/test/queries/clientpositive/groupby4_map.q (revision 0) +++ src/contrib/hive/ql/src/test/queries/clientpositive/groupby4_map.q (revision 0) @@ -0,0 +1,10 @@ +set hive.map.aggr=true; + +CREATE TABLE dest1(key INT) STORED AS TEXTFILE; + +EXPLAIN +FROM src INSERT OVERWRITE TABLE dest1 SELECT count(1); + +FROM src INSERT OVERWRITE TABLE dest1 SELECT count(1); + +SELECT dest1.* FROM dest1; Index: src/contrib/hive/ql/src/test/queries/clientpositive/inputddl5.q =================================================================== --- src/contrib/hive/ql/src/test/queries/clientpositive/inputddl5.q (revision 711472) +++ src/contrib/hive/ql/src/test/queries/clientpositive/inputddl5.q (working copy) @@ -1,6 +1,6 @@ -- test for internationalization -- kv4.txt contains the utf-8 character 0xE982B5E993AE which we are verifying later on -CREATE TABLE INPUTDDL5(name STRING); +CREATE TABLE INPUTDDL5(name STRING) STORED AS TEXTFILE; LOAD DATA LOCAL INPATH '../data/files/kv4.txt' INTO TABLE INPUTDDL5; DESCRIBE INPUTDDL5; SELECT INPUTDDL5.name from INPUTDDL5; Index: src/contrib/hive/ql/src/test/queries/clientpositive/inputddl7.q =================================================================== --- src/contrib/hive/ql/src/test/queries/clientpositive/inputddl7.q (revision 0) +++ src/contrib/hive/ql/src/test/queries/clientpositive/inputddl7.q (revision 0) @@ -0,0 +1,27 @@ +-- test for loading into tables with the correct file format +-- test for loading into partitions with the correct file format + +DROP TABLE T1; +CREATE TABLE T1(name STRING) STORED AS TEXTFILE; +LOAD DATA LOCAL INPATH '../data/files/kv1.txt' INTO TABLE T1; +SELECT COUNT(1) FROM T1; + +DROP TABLE T2; +CREATE TABLE T2(name STRING) STORED AS SEQUENCEFILE; +LOAD DATA LOCAL INPATH '../data/files/kv1.seq' INTO TABLE T2; +SELECT COUNT(1) FROM T2; + +DROP TABLE T3; +CREATE TABLE T3(name STRING) PARTITIONED BY(ds STRING) STORED AS TEXTFILE; +LOAD DATA LOCAL INPATH '../data/files/kv1.txt' INTO TABLE T3 PARTITION (ds='2008-04-09'); +SELECT COUNT(1) FROM T3 where T3.ds='2008-04-09'; + +DROP TABLE T4; +CREATE TABLE T4(name STRING) PARTITIONED BY(ds STRING) STORED AS SEQUENCEFILE; +LOAD DATA LOCAL INPATH '../data/files/kv1.seq' INTO TABLE T4 PARTITION (ds='2008-04-09'); +SELECT COUNT(1) FROM T4 where T4.ds='2008-04-09'; + +DESCRIBE EXTENDED T1; +DESCRIBE EXTENDED T2; +DESCRIBE EXTENDED T3 PARTITION (ds='2008-04-09'); +DESCRIBE EXTENDED T4 PARTITION (ds='2008-04-09'); Index: src/contrib/hive/ql/src/test/queries/clientpositive/notable_alias1.q =================================================================== --- src/contrib/hive/ql/src/test/queries/clientpositive/notable_alias1.q (revision 0) +++ src/contrib/hive/ql/src/test/queries/clientpositive/notable_alias1.q (revision 0) @@ -0,0 +1,10 @@ +CREATE TABLE dest1(dummy STRING, key INT, value DOUBLE) STORED AS TEXTFILE; + +EXPLAIN +FROM src +INSERT OVERWRITE TABLE dest1 SELECT '1234', key, count(1) WHERE src.key < 100 group by key; + +FROM src +INSERT OVERWRITE TABLE dest1 SELECT '1234', key, count(1) WHERE src.key < 100 group by key; + +SELECT dest1.* FROM dest1; Index: src/contrib/hive/ql/src/test/queries/clientpositive/join1.q =================================================================== --- src/contrib/hive/ql/src/test/queries/clientpositive/join1.q (revision 711472) +++ src/contrib/hive/ql/src/test/queries/clientpositive/join1.q (working copy) @@ -1,4 +1,4 @@ -CREATE TABLE dest1(key INT, value STRING); +CREATE TABLE dest1(key INT, value STRING) STORED AS TEXTFILE; EXPLAIN FROM src src1 JOIN src src2 ON (src1.key = src2.key) Index: src/contrib/hive/ql/src/test/queries/clientpositive/input2.q =================================================================== --- src/contrib/hive/ql/src/test/queries/clientpositive/input2.q (revision 711472) +++ src/contrib/hive/ql/src/test/queries/clientpositive/input2.q (working copy) @@ -1,6 +1,8 @@ -CREATE TABLE TEST2a(A INT, B FLOAT); +DROP TABLE TEST2a; +CREATE TABLE TEST2a(A INT, B FLOAT) STORED AS TEXTFILE; DESCRIBE TEST2a; -CREATE TABLE TEST2b(A ARRAY, B FLOAT, C MAP); +DROP TABLE TEST2b; +CREATE TABLE TEST2b(A ARRAY, B FLOAT, C MAP) STORED AS TEXTFILE; DESCRIBE TEST2b; SHOW TABLES; DROP TABLE TEST2a; Index: src/contrib/hive/ql/src/test/queries/clientpositive/join3.q =================================================================== --- src/contrib/hive/ql/src/test/queries/clientpositive/join3.q (revision 711472) +++ src/contrib/hive/ql/src/test/queries/clientpositive/join3.q (working copy) @@ -1,4 +1,4 @@ -CREATE TABLE dest1(key INT, value STRING); +CREATE TABLE dest1(key INT, value STRING) STORED AS TEXTFILE; EXPLAIN FROM src src1 JOIN src src2 ON (src1.key = src2.key) JOIN src src3 ON (src1.key = src3.key) Index: src/contrib/hive/ql/src/test/queries/clientpositive/input4.q =================================================================== --- src/contrib/hive/ql/src/test/queries/clientpositive/input4.q (revision 711472) +++ src/contrib/hive/ql/src/test/queries/clientpositive/input4.q (working copy) @@ -1,4 +1,4 @@ -CREATE TABLE INPUT4(KEY STRING, VALUE STRING); +CREATE TABLE INPUT4(KEY STRING, VALUE STRING) STORED AS TEXTFILE; EXPLAIN LOAD DATA LOCAL INPATH '../data/files/kv1.txt' INTO TABLE INPUT4; LOAD DATA LOCAL INPATH '../data/files/kv1.txt' INTO TABLE INPUT4; Index: src/contrib/hive/ql/src/test/queries/clientpositive/join5.q =================================================================== --- src/contrib/hive/ql/src/test/queries/clientpositive/join5.q (revision 711472) +++ src/contrib/hive/ql/src/test/queries/clientpositive/join5.q (working copy) @@ -1,4 +1,4 @@ -CREATE TABLE dest1(c1 INT, c2 STRING, c3 INT, c4 STRING); +CREATE TABLE dest1(c1 INT, c2 STRING, c3 INT, c4 STRING) STORED AS TEXTFILE; EXPLAIN FROM ( Index: src/contrib/hive/ql/src/test/queries/clientpositive/input_testxpath2.q =================================================================== --- src/contrib/hive/ql/src/test/queries/clientpositive/input_testxpath2.q (revision 711472) +++ src/contrib/hive/ql/src/test/queries/clientpositive/input_testxpath2.q (working copy) @@ -1,4 +1,4 @@ -CREATE TABLE dest1(lint_size INT, lintstring_size INT, mstringstring_size INT); +CREATE TABLE dest1(lint_size INT, lintstring_size INT, mstringstring_size INT) STORED AS TEXTFILE; EXPLAIN FROM src_thrift Index: src/contrib/hive/ql/src/test/queries/clientpositive/input6.q =================================================================== --- src/contrib/hive/ql/src/test/queries/clientpositive/input6.q (revision 711472) +++ src/contrib/hive/ql/src/test/queries/clientpositive/input6.q (working copy) @@ -1,4 +1,4 @@ -CREATE TABLE dest1(key STRING, value STRING); +CREATE TABLE dest1(key STRING, value STRING) STORED AS TEXTFILE; EXPLAIN FROM src1 Index: src/contrib/hive/ql/src/test/queries/clientpositive/join7.q =================================================================== --- src/contrib/hive/ql/src/test/queries/clientpositive/join7.q (revision 711472) +++ src/contrib/hive/ql/src/test/queries/clientpositive/join7.q (working copy) @@ -1,4 +1,4 @@ -CREATE TABLE dest1(c1 INT, c2 STRING, c3 INT, c4 STRING, c5 INT, c6 STRING); +CREATE TABLE dest1(c1 INT, c2 STRING, c3 INT, c4 STRING, c5 INT, c6 STRING) STORED AS TEXTFILE; EXPLAIN FROM ( Index: src/contrib/hive/ql/src/test/queries/clientpositive/input8.q =================================================================== --- src/contrib/hive/ql/src/test/queries/clientpositive/input8.q (revision 711472) +++ src/contrib/hive/ql/src/test/queries/clientpositive/input8.q (working copy) @@ -1,4 +1,4 @@ -CREATE TABLE dest1(c1 STRING, c2 INT, c3 DOUBLE); +CREATE TABLE dest1(c1 STRING, c2 INT, c3 DOUBLE) STORED AS TEXTFILE; EXPLAIN FROM src1 Index: src/contrib/hive/ql/src/test/queries/clientpositive/join9.q =================================================================== --- src/contrib/hive/ql/src/test/queries/clientpositive/join9.q (revision 711472) +++ src/contrib/hive/ql/src/test/queries/clientpositive/join9.q (working copy) @@ -1,4 +1,4 @@ -CREATE TABLE dest1(key INT, value STRING); +CREATE TABLE dest1(key INT, value STRING) STORED AS TEXTFILE; EXPLAIN EXTENDED FROM srcpart src1 JOIN src src2 ON (src1.key = src2.key) Index: src/contrib/hive/ql/src/test/queries/clientpositive/input10.q =================================================================== --- src/contrib/hive/ql/src/test/queries/clientpositive/input10.q (revision 711472) +++ src/contrib/hive/ql/src/test/queries/clientpositive/input10.q (working copy) @@ -1,4 +1,4 @@ -CREATE TABLE TEST10(key INT, value STRING) PARTITIONED BY(ds STRING, hr STRING); +CREATE TABLE TEST10(key INT, value STRING) PARTITIONED BY(ds STRING, hr STRING) STORED AS TEXTFILE; EXPLAIN DESCRIBE TEST10; Index: src/contrib/hive/ql/src/test/queries/clientpositive/udf2.q =================================================================== --- src/contrib/hive/ql/src/test/queries/clientpositive/udf2.q (revision 711472) +++ src/contrib/hive/ql/src/test/queries/clientpositive/udf2.q (working copy) @@ -1,4 +1,4 @@ -CREATE TABLE dest1(c1 STRING); +CREATE TABLE dest1(c1 STRING) STORED AS TEXTFILE; FROM src INSERT OVERWRITE TABLE dest1 SELECT ' abc ' WHERE src.key = 86; Index: src/contrib/hive/ql/src/test/queries/clientpositive/input4_cb_delim.q =================================================================== --- src/contrib/hive/ql/src/test/queries/clientpositive/input4_cb_delim.q (revision 711472) +++ src/contrib/hive/ql/src/test/queries/clientpositive/input4_cb_delim.q (working copy) @@ -1,4 +1,4 @@ -CREATE TABLE INPUT4_CB(KEY STRING, VALUE STRING) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\002' LINES TERMINATED BY '\012'; +CREATE TABLE INPUT4_CB(KEY STRING, VALUE STRING) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\002' LINES TERMINATED BY '\012' STORED AS TEXTFILE; LOAD DATA LOCAL INPATH '../data/files/kv1_cb.txt' INTO TABLE INPUT4_CB; SELECT INPUT4_CB.VALUE, INPUT4_CB.KEY FROM INPUT4_CB; DROP TABLE INPUT4_CB Index: src/contrib/hive/ql/src/test/queries/clientpositive/udf4.q =================================================================== --- src/contrib/hive/ql/src/test/queries/clientpositive/udf4.q (revision 711472) +++ src/contrib/hive/ql/src/test/queries/clientpositive/udf4.q (working copy) @@ -1,4 +1,4 @@ -CREATE TABLE dest1(c1 STRING); +CREATE TABLE dest1(c1 STRING) STORED AS TEXTFILE; FROM src INSERT OVERWRITE TABLE dest1 SELECT ' abc ' WHERE src.key = 86; Index: src/contrib/hive/ql/src/test/queries/clientpositive/input12.q =================================================================== --- src/contrib/hive/ql/src/test/queries/clientpositive/input12.q (revision 711472) +++ src/contrib/hive/ql/src/test/queries/clientpositive/input12.q (working copy) @@ -1,6 +1,6 @@ -CREATE TABLE dest1(key INT, value STRING); -CREATE TABLE dest2(key INT, value STRING); -CREATE TABLE dest3(key INT, value STRING) PARTITIONED BY(ds STRING, hr STRING); +CREATE TABLE dest1(key INT, value STRING) STORED AS TEXTFILE; +CREATE TABLE dest2(key INT, value STRING) STORED AS TEXTFILE; +CREATE TABLE dest3(key INT) PARTITIONED BY(ds STRING, hr STRING) STORED AS TEXTFILE; EXPLAIN FROM src Index: src/contrib/hive/ql/src/test/queries/clientpositive/input14.q =================================================================== --- src/contrib/hive/ql/src/test/queries/clientpositive/input14.q (revision 711472) +++ src/contrib/hive/ql/src/test/queries/clientpositive/input14.q (working copy) @@ -1,4 +1,4 @@ -CREATE TABLE dest1(key INT, value STRING); +CREATE TABLE dest1(key INT, value STRING) STORED AS TEXTFILE; EXPLAIN FROM ( Index: src/contrib/hive/ql/src/test/queries/clientpositive/input_part1.q =================================================================== --- src/contrib/hive/ql/src/test/queries/clientpositive/input_part1.q (revision 711472) +++ src/contrib/hive/ql/src/test/queries/clientpositive/input_part1.q (working copy) @@ -1,4 +1,4 @@ -CREATE TABLE dest1(key INT, value STRING, hr STRING, ds STRING); +CREATE TABLE dest1(key INT, value STRING, hr STRING, ds STRING) STORED AS TEXTFILE; EXPLAIN EXTENDED FROM srcpart Index: src/contrib/hive/ql/src/test/queries/clientpositive/input16.q =================================================================== --- src/contrib/hive/ql/src/test/queries/clientpositive/input16.q (revision 711472) +++ src/contrib/hive/ql/src/test/queries/clientpositive/input16.q (working copy) @@ -1,5 +1,5 @@ -- TestSerDe is a user defined serde where the default delimiter is Ctrl-B -CREATE TABLE INPUT16(KEY STRING, VALUE STRING) ROW FORMAT SERIALIZER 'org.apache.hadoop.hive.serde2.TestSerDe'; +CREATE TABLE INPUT16(KEY STRING, VALUE STRING) ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.TestSerDe' STORED AS TEXTFILE; LOAD DATA LOCAL INPATH '../data/files/kv1_cb.txt' INTO TABLE INPUT16; SELECT INPUT16.VALUE, INPUT16.KEY FROM INPUT16; DROP TABLE INPUT16; Index: src/contrib/hive/ql/src/test/queries/clientpositive/input_part3.q =================================================================== --- src/contrib/hive/ql/src/test/queries/clientpositive/input_part3.q (revision 0) +++ src/contrib/hive/ql/src/test/queries/clientpositive/input_part3.q (revision 0) @@ -0,0 +1,4 @@ +EXPLAIN +SELECT x.* FROM SRCPART x WHERE x.ds = '2008-04-08' and x.hr = 11; + +SELECT x.* FROM SRCPART x WHERE x.ds = '2008-04-08' and x.hr = 11; Index: src/contrib/hive/ql/src/test/queries/clientpositive/groupby2.q =================================================================== --- src/contrib/hive/ql/src/test/queries/clientpositive/groupby2.q (revision 711472) +++ src/contrib/hive/ql/src/test/queries/clientpositive/groupby2.q (working copy) @@ -1,4 +1,4 @@ -CREATE TABLE dest1(key STRING, c1 INT, c2 STRING); +CREATE TABLE dest1(key STRING, c1 INT, c2 STRING) STORED AS TEXTFILE; EXPLAIN FROM src Index: src/contrib/hive/ql/src/test/queries/clientpositive/input_part5.q =================================================================== --- src/contrib/hive/ql/src/test/queries/clientpositive/input_part5.q (revision 0) +++ src/contrib/hive/ql/src/test/queries/clientpositive/input_part5.q (revision 0) @@ -0,0 +1,4 @@ +EXPLAIN +SELECT x.* FROM SRCPART x WHERE x.ds = '2008-04-08' and x.key < 100; + +SELECT x.* FROM SRCPART x WHERE x.ds = '2008-04-08' and x.key < 100; Index: src/contrib/hive/ql/src/test/queries/clientpositive/show_tables.q =================================================================== --- src/contrib/hive/ql/src/test/queries/clientpositive/show_tables.q (revision 711472) +++ src/contrib/hive/ql/src/test/queries/clientpositive/show_tables.q (working copy) @@ -1,5 +1,5 @@ -CREATE TABLE shtb_test1(KEY STRING, VALUE STRING) PARTITIONED BY(ds DATETIME); -CREATE TABLE shtb_test2(KEY STRING, VALUE STRING) PARTITIONED BY(ds DATETIME); +CREATE TABLE shtb_test1(KEY STRING, VALUE STRING) PARTITIONED BY(ds DATETIME) STORED AS TEXTFILE; +CREATE TABLE shtb_test2(KEY STRING, VALUE STRING) PARTITIONED BY(ds DATETIME) STORED AS TEXTFILE; EXPLAIN SHOW TABLES 'shtb_*'; Index: src/contrib/hive/ql/src/test/queries/clientpositive/groupby4.q =================================================================== --- src/contrib/hive/ql/src/test/queries/clientpositive/groupby4.q (revision 711472) +++ src/contrib/hive/ql/src/test/queries/clientpositive/groupby4.q (working copy) @@ -1,4 +1,4 @@ -CREATE TABLE dest1(c1 STRING); +CREATE TABLE dest1(c1 STRING) STORED AS TEXTFILE; EXPLAIN FROM src Index: src/contrib/hive/ql/src/test/queries/clientpositive/groupby6.q =================================================================== --- src/contrib/hive/ql/src/test/queries/clientpositive/groupby6.q (revision 711472) +++ src/contrib/hive/ql/src/test/queries/clientpositive/groupby6.q (working copy) @@ -1,4 +1,4 @@ -CREATE TABLE dest1(c1 STRING); +CREATE TABLE dest1(c1 STRING) STORED AS TEXTFILE; EXPLAIN FROM src Index: src/contrib/hive/ql/src/test/queries/clientpositive/input1_limit.q =================================================================== --- src/contrib/hive/ql/src/test/queries/clientpositive/input1_limit.q (revision 711472) +++ src/contrib/hive/ql/src/test/queries/clientpositive/input1_limit.q (working copy) @@ -1,5 +1,5 @@ -CREATE TABLE dest1(key INT, value STRING); -CREATE TABLE dest2(key INT, value STRING); +CREATE TABLE dest1(key INT, value STRING) STORED AS TEXTFILE; +CREATE TABLE dest2(key INT, value STRING) STORED AS TEXTFILE; EXPLAIN FROM src Index: src/contrib/hive/ql/src/test/queries/clientpositive/groupby8.q =================================================================== --- src/contrib/hive/ql/src/test/queries/clientpositive/groupby8.q (revision 711472) +++ src/contrib/hive/ql/src/test/queries/clientpositive/groupby8.q (working copy) @@ -1,5 +1,5 @@ -CREATE TABLE DEST1(key INT, value STRING); -CREATE TABLE DEST2(key INT, value STRING); +CREATE TABLE DEST1(key INT, value STRING) STORED AS TEXTFILE; +CREATE TABLE DEST2(key INT, value STRING) STORED AS TEXTFILE; FROM SRC INSERT OVERWRITE TABLE DEST1 SELECT SRC.key, COUNT(DISTINCT SUBSTR(SRC.value,4)) GROUP BY SRC.key Index: src/contrib/hive/ql/src/test/queries/clientpositive/case_sensitivity.q =================================================================== --- src/contrib/hive/ql/src/test/queries/clientpositive/case_sensitivity.q (revision 711472) +++ src/contrib/hive/ql/src/test/queries/clientpositive/case_sensitivity.q (working copy) @@ -1,4 +1,4 @@ -CREATE TABLE DEST1(Key INT, VALUE STRING); +CREATE TABLE DEST1(Key INT, VALUE STRING) STORED AS TEXTFILE; EXPLAIN FROM SRC_THRIFT Index: src/contrib/hive/ql/src/test/queries/clientpositive/scriptfile1.q =================================================================== --- src/contrib/hive/ql/src/test/queries/clientpositive/scriptfile1.q (revision 0) +++ src/contrib/hive/ql/src/test/queries/clientpositive/scriptfile1.q (revision 0) @@ -0,0 +1,13 @@ +CREATE TABLE dest1(key INT, value STRING); + +ADD FILE src/test/scripts/testgrep; + +FROM ( + FROM src + SELECT TRANSFORM(src.key, src.value) AS (tkey, tvalue) + USING 'testgrep' + CLUSTER BY tkey +) tmap +INSERT OVERWRITE TABLE dest1 SELECT tmap.tkey, tmap.tvalue; + +SELECT dest1.* FROM dest1; Index: src/contrib/hive/ql/src/test/queries/clientpositive/sample1.q =================================================================== --- src/contrib/hive/ql/src/test/queries/clientpositive/sample1.q (revision 711472) +++ src/contrib/hive/ql/src/test/queries/clientpositive/sample1.q (working copy) @@ -1,13 +1,13 @@ -CREATE TABLE dest1(key INT, value STRING, dt STRING, hr STRING); +CREATE TABLE dest1(key INT, value STRING, dt STRING, hr STRING) STORED AS TEXTFILE; -- no input pruning, no sample filter EXPLAIN EXTENDED INSERT OVERWRITE TABLE dest1 SELECT s.* -FROM srcpart TABLESAMPLE (BUCKET 1 OUT OF 1) s +FROM srcpart TABLESAMPLE (BUCKET 1 OUT OF 1 ON rand()) s WHERE s.ds='2008-04-08' and s.hr='11'; INSERT OVERWRITE TABLE dest1 SELECT s.* -FROM srcpart TABLESAMPLE (BUCKET 1 OUT OF 1) s +FROM srcpart TABLESAMPLE (BUCKET 1 OUT OF 1 ON rand()) s WHERE s.ds='2008-04-08' and s.hr='11'; SELECT dest1.* FROM dest1; Index: src/contrib/hive/ql/src/test/queries/clientpositive/groupby1_map.q =================================================================== --- src/contrib/hive/ql/src/test/queries/clientpositive/groupby1_map.q (revision 0) +++ src/contrib/hive/ql/src/test/queries/clientpositive/groupby1_map.q (revision 0) @@ -0,0 +1,10 @@ +set hive.map.aggr=true; + +CREATE TABLE dest1(key INT, value DOUBLE) STORED AS TEXTFILE; + +EXPLAIN +FROM src INSERT OVERWRITE TABLE dest1 SELECT src.key, sum(substr(src.value,4)) GROUP BY src.key; + +FROM src INSERT OVERWRITE TABLE dest1 SELECT src.key, sum(substr(src.value,4)) GROUP BY src.key; + +SELECT dest1.* FROM dest1; Property changes on: src/contrib/hive/ql/src/test/queries/clientpositive/groupby1_map.q ___________________________________________________________________ Added: svn:executable + * Index: src/contrib/hive/ql/src/test/queries/clientpositive/inputddl2.q =================================================================== --- src/contrib/hive/ql/src/test/queries/clientpositive/inputddl2.q (revision 711472) +++ src/contrib/hive/ql/src/test/queries/clientpositive/inputddl2.q (working copy) @@ -1,6 +1,6 @@ EXPLAIN -CREATE TABLE INPUTDDL2(key INT, value STRING) PARTITIONED BY(ds DATETIME, country STRING); -CREATE TABLE INPUTDDL2(key INT, value STRING) PARTITIONED BY(ds DATETIME, country STRING); +CREATE TABLE INPUTDDL2(key INT, value STRING) PARTITIONED BY(ds DATETIME, country STRING) STORED AS TEXTFILE; +CREATE TABLE INPUTDDL2(key INT, value STRING) PARTITIONED BY(ds DATETIME, country STRING) STORED AS TEXTFILE; DESCRIBE INPUTDDL2; DROP TABLE INPUTDDL2; Index: src/contrib/hive/ql/src/test/queries/clientpositive/groupby1_limit.q =================================================================== --- src/contrib/hive/ql/src/test/queries/clientpositive/groupby1_limit.q (revision 711472) +++ src/contrib/hive/ql/src/test/queries/clientpositive/groupby1_limit.q (working copy) @@ -1,6 +1,6 @@ set mapred.reduce.tasks=31; -CREATE TABLE dest1(key INT, value DOUBLE); +CREATE TABLE dest1(key INT, value DOUBLE) STORED AS TEXTFILE; EXPLAIN FROM src INSERT OVERWRITE TABLE dest1 SELECT src.key, sum(substr(src.value,4)) GROUP BY src.key LIMIT 5; Index: src/contrib/hive/ql/src/test/queries/clientpositive/sample5.q =================================================================== --- src/contrib/hive/ql/src/test/queries/clientpositive/sample5.q (revision 711472) +++ src/contrib/hive/ql/src/test/queries/clientpositive/sample5.q (working copy) @@ -1,4 +1,4 @@ -CREATE TABLE dest1(key INT, value STRING); +CREATE TABLE dest1(key INT, value STRING) STORED AS TEXTFILE; -- no input pruning, sample filter EXPLAIN EXTENDED Index: src/contrib/hive/ql/src/test/queries/clientpositive/groupby3_map.q =================================================================== --- src/contrib/hive/ql/src/test/queries/clientpositive/groupby3_map.q (revision 0) +++ src/contrib/hive/ql/src/test/queries/clientpositive/groupby3_map.q (revision 0) @@ -0,0 +1,12 @@ +set hive.map.aggr=true; + +CREATE TABLE dest1(c1 INT, c2 INT, c3 INT, c4 INT, c5 INT) STORED AS TEXTFILE; + +EXPLAIN +FROM src +INSERT OVERWRITE TABLE dest1 SELECT sum(substr(src.value,4)), avg(substr(src.value,4)), avg(DISTINCT substr(src.value,4)), max(substr(src.value,4)), min(substr(src.value,4)); + +FROM src +INSERT OVERWRITE TABLE dest1 SELECT sum(substr(src.value,4)), avg(substr(src.value,4)), avg(DISTINCT substr(src.value,4)), max(substr(src.value,4)), min(substr(src.value,4)); + +SELECT dest1.* FROM dest1; Index: src/contrib/hive/ql/src/test/queries/clientpositive/sample7.q =================================================================== --- src/contrib/hive/ql/src/test/queries/clientpositive/sample7.q (revision 711472) +++ src/contrib/hive/ql/src/test/queries/clientpositive/sample7.q (working copy) @@ -1,4 +1,4 @@ -CREATE TABLE dest1(key INT, value STRING); +CREATE TABLE dest1(key INT, value STRING) STORED AS TEXTFILE; -- both input pruning and sample filter EXPLAIN EXTENDED Index: src/contrib/hive/ql/src/test/queries/clientpositive/groupby5_map.q =================================================================== --- src/contrib/hive/ql/src/test/queries/clientpositive/groupby5_map.q (revision 0) +++ src/contrib/hive/ql/src/test/queries/clientpositive/groupby5_map.q (revision 0) @@ -0,0 +1,10 @@ +set hive.map.aggr=true; + +CREATE TABLE dest1(key INT) STORED AS TEXTFILE; + +EXPLAIN +FROM src INSERT OVERWRITE TABLE dest1 SELECT sum(src.key); + +FROM src INSERT OVERWRITE TABLE dest1 SELECT sum(src.key); + +SELECT dest1.* FROM dest1; Index: src/contrib/hive/ql/src/test/queries/clientpositive/inputddl6.q =================================================================== --- src/contrib/hive/ql/src/test/queries/clientpositive/inputddl6.q (revision 711472) +++ src/contrib/hive/ql/src/test/queries/clientpositive/inputddl6.q (working copy) @@ -2,7 +2,7 @@ -- test for describe extended table partition -- test for alter table drop partition DROP TABLE INPUTDDL6; -CREATE TABLE INPUTDDL6(KEY STRING, VALUE STRING) PARTITIONED BY(ds DATETIME); +CREATE TABLE INPUTDDL6(KEY STRING, VALUE STRING) PARTITIONED BY(ds DATETIME) STORED AS TEXTFILE; LOAD DATA LOCAL INPATH '../data/files/kv1.txt' INTO TABLE INPUTDDL6 PARTITION (ds='2008-04-09'); LOAD DATA LOCAL INPATH '../data/files/kv1.txt' INTO TABLE INPUTDDL6 PARTITION (ds='2008-04-08'); DESCRIBE EXTENDED INPUTDDL6; Index: src/contrib/hive/ql/src/test/queries/clientpositive/input16_cc.q =================================================================== --- src/contrib/hive/ql/src/test/queries/clientpositive/input16_cc.q (revision 711472) +++ src/contrib/hive/ql/src/test/queries/clientpositive/input16_cc.q (working copy) @@ -1,6 +1,7 @@ -- TestSerDe is a user defined serde where the default delimiter is Ctrl-B -- the user is overwriting it with ctrlC -CREATE TABLE INPUT16_CC(KEY STRING, VALUE STRING) ROW FORMAT SERIALIZER 'org.apache.hadoop.hive.serde2.TestSerDe' with serdeproperties ('testserde.default.serialization.format'='\003', 'dummy.prop.not.used'='dummyy.val'); +DROP TABLE INPUT16_CC; +CREATE TABLE INPUT16_CC(KEY STRING, VALUE STRING) ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.TestSerDe' with serdeproperties ('testserde.default.serialization.format'='\003', 'dummy.prop.not.used'='dummyy.val') STORED AS TEXTFILE; LOAD DATA LOCAL INPATH '../data/files/kv1_cc.txt' INTO TABLE INPUT16_CC; SELECT INPUT16_CC.VALUE, INPUT16_CC.KEY FROM INPUT16_CC; DROP TABLE INPUT16_CC; Index: src/contrib/hive/ql/src/test/queries/clientpositive/cast1.q =================================================================== --- src/contrib/hive/ql/src/test/queries/clientpositive/cast1.q (revision 711472) +++ src/contrib/hive/ql/src/test/queries/clientpositive/cast1.q (working copy) @@ -1,4 +1,4 @@ -CREATE TABLE dest1(c1 INT, c2 DOUBLE, c3 DOUBLE, c4 DOUBLE, c5 INT, c6 INT, c7 INT); +CREATE TABLE dest1(c1 INT, c2 DOUBLE, c3 DOUBLE, c4 DOUBLE, c5 INT, c6 INT, c7 INT) STORED AS TEXTFILE; EXPLAIN FROM src INSERT OVERWRITE TABLE dest1 SELECT 3 + 2, 3.0 + 2, 3 + 2.0, 3.0 + 2.0, 3 + CAST(2.0 AS INT), CAST(1 AS BOOLEAN), CAST(TRUE AS INT) WHERE src.key = 86; Index: src/contrib/hive/ql/src/test/queries/clientpositive/inputddl8.q =================================================================== --- src/contrib/hive/ql/src/test/queries/clientpositive/inputddl8.q (revision 0) +++ src/contrib/hive/ql/src/test/queries/clientpositive/inputddl8.q (revision 0) @@ -0,0 +1,8 @@ +DROP TABLE INPUTDDL8; +CREATE TABLE INPUTDDL8 COMMENT 'This is a thrift based table' + PARTITIONED BY(ds DATETIME, country STRING) + CLUSTERED BY(aint) SORTED BY(lint) INTO 32 BUCKETS + ROW FORMAT SERDE'org.apache.hadoop.hive.serde2.ThriftDeserializer' WITH SERDEPROPERTIES ('serialization.class' = 'org.apache.hadoop.hive.serde2.thrift.test.Complex', 'serialization.format' = 'com.facebook.thrift.protocol.TBinaryProtocol') + STORED AS SEQUENCEFILE; +DESCRIBE EXTENDED INPUTDDL8; +DROP TABLE INPUTDDL8; Index: src/contrib/hive/ql/src/test/queries/clientpositive/quote1.q =================================================================== --- src/contrib/hive/ql/src/test/queries/clientpositive/quote1.q (revision 0) +++ src/contrib/hive/ql/src/test/queries/clientpositive/quote1.q (revision 0) @@ -0,0 +1,13 @@ +CREATE TABLE dest1(`location` INT, `type` STRING) PARTITIONED BY(`table` STRING) STORED AS TEXTFILE; + +EXPLAIN +FROM src +INSERT OVERWRITE TABLE dest1 PARTITION(`table`='2008-04-08') SELECT src.key as `partition`, src.value as `from` WHERE src.key >= 200 and src.key < 300; + +EXPLAIN +SELECT `table`.`location`, `table`.`type`, `table`.`table` FROM dest1 `table` WHERE `table`.`table` = '2008-04-08'; + +FROM src +INSERT OVERWRITE TABLE dest1 PARTITION(`table`='2008-04-08') SELECT src.key as `partition`, src.value as `from` WHERE src.key >= 200 and src.key < 300; + +SELECT `table`.`location`, `table`.`type`, `table`.`table` FROM dest1 `table` WHERE `table`.`table` = '2008-04-08'; Index: src/contrib/hive/ql/src/test/queries/clientpositive/notable_alias2.q =================================================================== --- src/contrib/hive/ql/src/test/queries/clientpositive/notable_alias2.q (revision 0) +++ src/contrib/hive/ql/src/test/queries/clientpositive/notable_alias2.q (revision 0) @@ -0,0 +1,10 @@ +CREATE TABLE dest1(dummy STRING, key INT, value DOUBLE) STORED AS TEXTFILE; + +EXPLAIN +FROM src +INSERT OVERWRITE TABLE dest1 SELECT '1234', src.key, count(1) WHERE key < 100 group by src.key; + +FROM src +INSERT OVERWRITE TABLE dest1 SELECT '1234', src.key, count(1) WHERE key < 100 group by src.key; + +SELECT dest1.* FROM dest1; Index: src/contrib/hive/ql/src/test/queries/clientpositive/input1.q =================================================================== --- src/contrib/hive/ql/src/test/queries/clientpositive/input1.q (revision 711472) +++ src/contrib/hive/ql/src/test/queries/clientpositive/input1.q (working copy) @@ -1,4 +1,4 @@ -CREATE TABLE TEST1(A INT, B FLOAT); +CREATE TABLE TEST1(A INT, B FLOAT) STORED AS TEXTFILE; EXPLAIN DESCRIBE TEST1; Index: src/contrib/hive/ql/src/test/queries/clientpositive/join2.q =================================================================== --- src/contrib/hive/ql/src/test/queries/clientpositive/join2.q (revision 711472) +++ src/contrib/hive/ql/src/test/queries/clientpositive/join2.q (working copy) @@ -1,4 +1,4 @@ -CREATE TABLE dest1(key INT, value STRING); +CREATE TABLE dest1(key INT, value STRING) STORED AS TEXTFILE; EXPLAIN FROM src src1 JOIN src src2 ON (src1.key = src2.key) JOIN src src3 ON (src1.key + src2.key = src3.key) Index: src/contrib/hive/ql/src/test/queries/clientpositive/input3.q =================================================================== --- src/contrib/hive/ql/src/test/queries/clientpositive/input3.q (revision 711472) +++ src/contrib/hive/ql/src/test/queries/clientpositive/input3.q (working copy) @@ -1,9 +1,9 @@ DROP TABLE TEST3a; DROP TABLE TEST3b; DROP TABLE TEST3c; -CREATE TABLE TEST3a(A INT, B FLOAT); +CREATE TABLE TEST3a(A INT, B FLOAT) STORED AS TEXTFILE; DESCRIBE TEST3a; -CREATE TABLE TEST3b(A ARRAY, B FLOAT, C MAP); +CREATE TABLE TEST3b(A ARRAY, B FLOAT, C MAP) STORED AS TEXTFILE; DESCRIBE TEST3b; SHOW TABLES; EXPLAIN Index: src/contrib/hive/ql/src/test/queries/clientpositive/join4.q =================================================================== --- src/contrib/hive/ql/src/test/queries/clientpositive/join4.q (revision 711472) +++ src/contrib/hive/ql/src/test/queries/clientpositive/join4.q (working copy) @@ -1,4 +1,4 @@ -CREATE TABLE dest1(c1 INT, c2 STRING, c3 INT, c4 STRING); +CREATE TABLE dest1(c1 INT, c2 STRING, c3 INT, c4 STRING) STORED AS TEXTFILE; EXPLAIN FROM ( Index: src/contrib/hive/ql/src/test/queries/clientpositive/input5.q =================================================================== --- src/contrib/hive/ql/src/test/queries/clientpositive/input5.q (revision 711472) +++ src/contrib/hive/ql/src/test/queries/clientpositive/input5.q (working copy) @@ -1,4 +1,4 @@ -CREATE TABLE dest1(key INT, value STRING); +CREATE TABLE dest1(key INT, value STRING) STORED AS TEXTFILE; EXPLAIN FROM ( Index: src/contrib/hive/ql/src/test/queries/clientpositive/join6.q =================================================================== --- src/contrib/hive/ql/src/test/queries/clientpositive/join6.q (revision 711472) +++ src/contrib/hive/ql/src/test/queries/clientpositive/join6.q (working copy) @@ -1,4 +1,4 @@ -CREATE TABLE dest1(c1 INT, c2 STRING, c3 INT, c4 STRING); +CREATE TABLE dest1(c1 INT, c2 STRING, c3 INT, c4 STRING) STORED AS TEXTFILE; EXPLAIN FROM ( Index: src/contrib/hive/ql/src/test/queries/clientpositive/input_dynamicserde.q =================================================================== --- src/contrib/hive/ql/src/test/queries/clientpositive/input_dynamicserde.q (revision 711472) +++ src/contrib/hive/ql/src/test/queries/clientpositive/input_dynamicserde.q (working copy) @@ -3,7 +3,8 @@ FIELDS TERMINATED BY '1' COLLECTION ITEMS TERMINATED BY '2' MAP KEYS TERMINATED BY '3' -LINES TERMINATED BY '10'; +LINES TERMINATED BY '10' +STORED AS TEXTFILE; EXPLAIN FROM src_thrift @@ -13,3 +14,5 @@ INSERT OVERWRITE TABLE dest1 SELECT src_thrift.lint, src_thrift.lstring, src_thrift.mstringstring, src_thrift.aint, src_thrift.astring; SELECT dest1.* FROM dest1; + +SELECT dest1.a[0], dest1.b[0], dest1.c['key2'], dest1.d, dest1.e FROM dest1; Index: src/contrib/hive/ql/src/test/queries/clientpositive/input7.q =================================================================== --- src/contrib/hive/ql/src/test/queries/clientpositive/input7.q (revision 711472) +++ src/contrib/hive/ql/src/test/queries/clientpositive/input7.q (working copy) @@ -1,4 +1,4 @@ -CREATE TABLE dest1(c1 DOUBLE, c2 INT); +CREATE TABLE dest1(c1 DOUBLE, c2 INT) STORED AS TEXTFILE; EXPLAIN FROM src1 Index: src/contrib/hive/ql/src/test/queries/clientpositive/join8.q =================================================================== --- src/contrib/hive/ql/src/test/queries/clientpositive/join8.q (revision 711472) +++ src/contrib/hive/ql/src/test/queries/clientpositive/join8.q (working copy) @@ -1,4 +1,4 @@ -CREATE TABLE dest1(c1 INT, c2 STRING, c3 INT, c4 STRING); +CREATE TABLE dest1(c1 INT, c2 STRING, c3 INT, c4 STRING) STORED AS TEXTFILE; EXPLAIN FROM ( Index: src/contrib/hive/ql/src/test/queries/clientpositive/input9.q =================================================================== --- src/contrib/hive/ql/src/test/queries/clientpositive/input9.q (revision 711472) +++ src/contrib/hive/ql/src/test/queries/clientpositive/input9.q (working copy) @@ -1,4 +1,4 @@ -CREATE TABLE dest1(value STRING, key INT); +CREATE TABLE dest1(value STRING, key INT) STORED AS TEXTFILE; EXPLAIN FROM src1 Index: src/contrib/hive/ql/src/test/queries/clientpositive/input_dfs.q =================================================================== --- src/contrib/hive/ql/src/test/queries/clientpositive/input_dfs.q (revision 711472) +++ src/contrib/hive/ql/src/test/queries/clientpositive/input_dfs.q (working copy) @@ -1,4 +1,2 @@ dfs -cat ../../../../build/contrib/hive/ql/test/data/files/kv1.txt; -set fs.default.name=file://src -dfs -ls Index: src/contrib/hive/ql/src/test/queries/clientpositive/udf1.q =================================================================== --- src/contrib/hive/ql/src/test/queries/clientpositive/udf1.q (revision 711472) +++ src/contrib/hive/ql/src/test/queries/clientpositive/udf1.q (working copy) @@ -1,7 +1,7 @@ CREATE TABLE dest1(c1 STRING, c2 STRING, c3 STRING, c4 STRING, c5 STRING, c6 STRING, c7 STRING, c8 STRING, c9 STRING, c10 STRING, c11 STRING, c12 STRING, c13 STRING, - c14 STRING, c15 STRING, c16 STRING, c17 STRING); + c14 STRING, c15 STRING, c16 STRING, c17 STRING) STORED AS TEXTFILE; EXPLAIN FROM src INSERT OVERWRITE TABLE dest1 SELECT 'a' LIKE '%a%', 'b' LIKE '%a%', 'ab' LIKE '%a%', 'ab' LIKE '%a_', Index: src/contrib/hive/ql/src/test/queries/clientpositive/input11.q =================================================================== --- src/contrib/hive/ql/src/test/queries/clientpositive/input11.q (revision 711472) +++ src/contrib/hive/ql/src/test/queries/clientpositive/input11.q (working copy) @@ -1,4 +1,4 @@ -CREATE TABLE dest1(key INT, value STRING); +CREATE TABLE dest1(key INT, value STRING) STORED AS TEXTFILE; EXPLAIN FROM src Index: src/contrib/hive/ql/src/test/queries/clientpositive/udf3.q =================================================================== --- src/contrib/hive/ql/src/test/queries/clientpositive/udf3.q (revision 711472) +++ src/contrib/hive/ql/src/test/queries/clientpositive/udf3.q (working copy) @@ -1,4 +1,4 @@ -CREATE TABLE dest1(c1 STRING, c2 STRING, c3 STRING, c4 STRING, c5 STRING); +CREATE TABLE dest1(c1 STRING, c2 STRING, c3 STRING, c4 STRING, c5 STRING) STORED AS TEXTFILE; EXPLAIN FROM src INSERT OVERWRITE TABLE dest1 SELECT count(CAST('' AS INT)), sum(CAST('' AS INT)), avg(CAST('' AS INT)), Index: src/contrib/hive/ql/src/test/queries/clientpositive/noalias_subq1.q =================================================================== --- src/contrib/hive/ql/src/test/queries/clientpositive/noalias_subq1.q (revision 0) +++ src/contrib/hive/ql/src/test/queries/clientpositive/noalias_subq1.q (revision 0) @@ -0,0 +1,5 @@ +EXPLAIN +SELECT c1 FROM (select value as c1, key as c2 from src) x where c2 < 100; + +SELECT c1 FROM (select value as c1, key as c2 from src) x where c2 < 100; + Index: src/contrib/hive/ql/src/test/queries/clientpositive/input_testxpath.q =================================================================== --- src/contrib/hive/ql/src/test/queries/clientpositive/input_testxpath.q (revision 711472) +++ src/contrib/hive/ql/src/test/queries/clientpositive/input_testxpath.q (working copy) @@ -1,4 +1,4 @@ -CREATE TABLE dest1(key INT, value STRING, mapvalue STRING); +CREATE TABLE dest1(key INT, value STRING, mapvalue STRING) STORED AS TEXTFILE; EXPLAIN FROM src_thrift Index: src/contrib/hive/ql/src/test/queries/clientpositive/input13.q =================================================================== --- src/contrib/hive/ql/src/test/queries/clientpositive/input13.q (revision 711472) +++ src/contrib/hive/ql/src/test/queries/clientpositive/input13.q (working copy) @@ -1,6 +1,6 @@ -CREATE TABLE dest1(key INT, value STRING); -CREATE TABLE dest2(key INT, value STRING); -CREATE TABLE dest3(key INT, value STRING) PARTITIONED BY(ds STRING, hr STRING); +CREATE TABLE dest1(key INT, value STRING) STORED AS TEXTFILE; +CREATE TABLE dest2(key INT, value STRING) STORED AS TEXTFILE; +CREATE TABLE dest3(key INT) PARTITIONED BY(ds STRING, hr STRING) STORED AS TEXTFILE; EXPLAIN FROM src Index: src/contrib/hive/ql/src/test/queries/clientpositive/input15.q =================================================================== --- src/contrib/hive/ql/src/test/queries/clientpositive/input15.q (revision 711472) +++ src/contrib/hive/ql/src/test/queries/clientpositive/input15.q (working copy) @@ -1,7 +1,7 @@ EXPLAIN -CREATE TABLE TEST15(key INT, value STRING) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t'; +CREATE TABLE TEST15(key INT, value STRING) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t' STORED AS TEXTFILE; -CREATE TABLE TEST15(key INT, value STRING) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t'; +CREATE TABLE TEST15(key INT, value STRING) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t' STORED AS TEXTFILE; DESCRIBE TEST15; Index: src/contrib/hive/ql/src/test/queries/clientpositive/input_part0.q =================================================================== --- src/contrib/hive/ql/src/test/queries/clientpositive/input_part0.q (revision 0) +++ src/contrib/hive/ql/src/test/queries/clientpositive/input_part0.q (revision 0) @@ -0,0 +1,4 @@ +EXPLAIN +SELECT x.* FROM SRCPART x WHERE x.ds = '2008-04-08'; + +SELECT x.* FROM SRCPART x WHERE x.ds = '2008-04-08'; Index: src/contrib/hive/ql/src/test/queries/clientpositive/input_part2.q =================================================================== --- src/contrib/hive/ql/src/test/queries/clientpositive/input_part2.q (revision 711472) +++ src/contrib/hive/ql/src/test/queries/clientpositive/input_part2.q (working copy) @@ -1,5 +1,5 @@ -CREATE TABLE dest1(key INT, value STRING, hr STRING, ds STRING); -CREATE TABLE dest2(key INT, value STRING, hr STRING, ds STRING); +CREATE TABLE dest1(key INT, value STRING, hr STRING, ds STRING) STORED AS TEXTFILE; +CREATE TABLE dest2(key INT, value STRING, hr STRING, ds STRING) STORED AS TEXTFILE; EXPLAIN EXTENDED FROM srcpart Index: src/contrib/hive/ql/src/test/queries/clientpositive/input17.q =================================================================== --- src/contrib/hive/ql/src/test/queries/clientpositive/input17.q (revision 0) +++ src/contrib/hive/ql/src/test/queries/clientpositive/input17.q (revision 0) @@ -0,0 +1,20 @@ +CREATE TABLE dest1(key INT, value STRING) STORED AS TEXTFILE; + +EXPLAIN +FROM ( + FROM src_thrift + SELECT TRANSFORM(src_thrift.aint + src_thrift.lint[0], src_thrift.lintstring[0]) AS (tkey, tvalue) + USING '/bin/cat' + CLUSTER BY tkey +) tmap +INSERT OVERWRITE TABLE dest1 SELECT tmap.tkey, tmap.tvalue; + +FROM ( + FROM src_thrift + SELECT TRANSFORM(src_thrift.aint + src_thrift.lint[0], src_thrift.lintstring[0]) AS (tkey, tvalue) + USING '/bin/cat' + CLUSTER BY tkey +) tmap +INSERT OVERWRITE TABLE dest1 SELECT tmap.tkey, tmap.tvalue; + +SELECT dest1.* FROM dest1; Index: src/contrib/hive/ql/src/test/queries/clientpositive/groupby1.q =================================================================== --- src/contrib/hive/ql/src/test/queries/clientpositive/groupby1.q (revision 711472) +++ src/contrib/hive/ql/src/test/queries/clientpositive/groupby1.q (working copy) @@ -1,4 +1,4 @@ -CREATE TABLE dest1(key INT, value DOUBLE); +CREATE TABLE dest1(key INT, value DOUBLE) STORED AS TEXTFILE; EXPLAIN FROM src INSERT OVERWRITE TABLE dest1 SELECT src.key, sum(substr(src.value,4)) GROUP BY src.key; Index: src/contrib/hive/ql/src/test/queries/clientpositive/input_part4.q =================================================================== --- src/contrib/hive/ql/src/test/queries/clientpositive/input_part4.q (revision 0) +++ src/contrib/hive/ql/src/test/queries/clientpositive/input_part4.q (revision 0) @@ -0,0 +1,4 @@ +EXPLAIN +SELECT x.* FROM SRCPART x WHERE x.ds = '2008-04-08' and x.hr = 15; + +SELECT x.* FROM SRCPART x WHERE x.ds = '2008-04-08' and x.hr = 15; Index: src/contrib/hive/ql/src/test/queries/clientpositive/groupby3.q =================================================================== --- src/contrib/hive/ql/src/test/queries/clientpositive/groupby3.q (revision 711472) +++ src/contrib/hive/ql/src/test/queries/clientpositive/groupby3.q (working copy) @@ -1,4 +1,4 @@ -CREATE TABLE dest1(c1 INT, c2 INT, c3 INT, c4 INT, c5 INT); +CREATE TABLE dest1(c1 INT, c2 INT, c3 INT, c4 INT, c5 INT) STORED AS TEXTFILE; EXPLAIN FROM src Index: src/contrib/hive/ql/src/test/queries/clientpositive/groupby5.q =================================================================== --- src/contrib/hive/ql/src/test/queries/clientpositive/groupby5.q (revision 711472) +++ src/contrib/hive/ql/src/test/queries/clientpositive/groupby5.q (working copy) @@ -1,4 +1,4 @@ -CREATE TABLE dest1(key INT, value STRING); +CREATE TABLE dest1(key INT, value STRING) STORED AS TEXTFILE; EXPLAIN INSERT OVERWRITE TABLE dest1 Index: src/contrib/hive/ql/src/test/queries/clientpositive/groupby7.q =================================================================== --- src/contrib/hive/ql/src/test/queries/clientpositive/groupby7.q (revision 711472) +++ src/contrib/hive/ql/src/test/queries/clientpositive/groupby7.q (working copy) @@ -1,5 +1,5 @@ -CREATE TABLE DEST1(key INT, value STRING); -CREATE TABLE DEST2(key INT, value STRING); +CREATE TABLE DEST1(key INT, value STRING) STORED AS TEXTFILE; +CREATE TABLE DEST2(key INT, value STRING) STORED AS TEXTFILE; FROM SRC INSERT OVERWRITE TABLE DEST1 SELECT SRC.key, sum(SUBSTR(SRC.value,4)) GROUP BY SRC.key Index: src/contrib/hive/ql/src/test/queries/negative/wrong_distinct3.q =================================================================== --- src/contrib/hive/ql/src/test/queries/negative/wrong_distinct3.q (revision 711472) +++ src/contrib/hive/ql/src/test/queries/negative/wrong_distinct3.q (working copy) @@ -1,3 +0,0 @@ -FROM SRC -INSERT OVERWRITE TABLE DEST1 SELECT SRC.key, COUNT(DISTINCT SUBSTR(SRC.value,4)) GROUP BY SRC.key -INSERT OVERWRITE TABLE DEST2 SELECT SRC.key, COUNT(DISTINCT SUBSTR(SRC.value,5)) GROUP BY SRC.key Index: src/contrib/hive/ql/src/test/queries/negative/notable_alias1.q =================================================================== --- src/contrib/hive/ql/src/test/queries/negative/notable_alias1.q (revision 711472) +++ src/contrib/hive/ql/src/test/queries/negative/notable_alias1.q (working copy) @@ -1,2 +0,0 @@ -FROM src -INSERT OVERWRITE TABLE dest1 SELECT '1234', value WHERE src.key < 100 group by src.key Index: src/contrib/hive/ql/src/test/queries/negative/notable_alias2.q =================================================================== --- src/contrib/hive/ql/src/test/queries/negative/notable_alias2.q (revision 711472) +++ src/contrib/hive/ql/src/test/queries/negative/notable_alias2.q (working copy) @@ -1,2 +0,0 @@ -FROM src -INSERT OVERWRITE TABLE dest1 SELECT '1234', src.value WHERE key < 100 group by src.key Index: src/contrib/hive/ql/src/test/queries/negative/notable_alias3.q =================================================================== --- src/contrib/hive/ql/src/test/queries/negative/notable_alias3.q (revision 711472) +++ src/contrib/hive/ql/src/test/queries/negative/notable_alias3.q (working copy) @@ -1,2 +0,0 @@ -FROM src -INSERT OVERWRITE TABLE dest1 SELECT '1234', src.value WHERE src.key < 100 group by key Index: src/contrib/hive/ql/src/test/queries/negative/quoted_string.q =================================================================== --- src/contrib/hive/ql/src/test/queries/negative/quoted_string.q (revision 711472) +++ src/contrib/hive/ql/src/test/queries/negative/quoted_string.q (working copy) @@ -1,2 +1,2 @@ FROM src -INSERT OVERWRITE TABLE dest1 SELECT "1234", src.value WHERE src.key < 100 +INSERT OVERWRITE TABLE dest1 SELECT '1234", src.value WHERE src.key < 100 Index: src/contrib/hive/ql/src/test/queries/negative/insert_wrong_number_columns.q =================================================================== --- src/contrib/hive/ql/src/test/queries/negative/insert_wrong_number_columns.q (revision 0) +++ src/contrib/hive/ql/src/test/queries/negative/insert_wrong_number_columns.q (revision 0) @@ -0,0 +1,2 @@ +FROM src +INSERT OVERWRITE TABLE dest1 SELECT src.key, src.value, 1 WHERE src.key < 100 Index: src/contrib/hive/ql/src/test/queries/positive/sample1.q =================================================================== --- src/contrib/hive/ql/src/test/queries/positive/sample1.q (revision 711472) +++ src/contrib/hive/ql/src/test/queries/positive/sample1.q (working copy) @@ -1,5 +1,5 @@ -- no input pruning, no sample filter -INSERT OVERWRITE TABLE dest1 SELECT s.* -FROM srcpart TABLESAMPLE (BUCKET 1 OUT OF 1) s +SELECT s.* +FROM srcpart TABLESAMPLE (BUCKET 1 OUT OF 1 ON rand()) s WHERE s.ds='2008-04-08' and s.hr='11' Index: src/contrib/hive/ql/src/test/queries/positive/cast1.q =================================================================== --- src/contrib/hive/ql/src/test/queries/positive/cast1.q (revision 711472) +++ src/contrib/hive/ql/src/test/queries/positive/cast1.q (working copy) @@ -1,2 +1,2 @@ FROM src -INSERT OVERWRITE TABLE dest1 SELECT 3 + 2, 3.0 + 2, 3 + 2.0, 3.0 + 2.0, 3 + CAST(2.0 AS INT), CAST(1 AS BOOLEAN), CAST(TRUE AS INT) WHERE src.key = 86 +SELECT 3 + 2, 3.0 + 2, 3 + 2.0, 3.0 + 2.0, 3 + CAST(2.0 AS INT), CAST(1 AS BOOLEAN), CAST(TRUE AS INT) WHERE src.key = 86 Index: src/contrib/hive/ql/src/test/queries/positive/input2.q =================================================================== --- src/contrib/hive/ql/src/test/queries/positive/input2.q (revision 711472) +++ src/contrib/hive/ql/src/test/queries/positive/input2.q (working copy) @@ -1,4 +1,4 @@ FROM src INSERT OVERWRITE TABLE dest1 SELECT src.* WHERE src.key < 100 INSERT OVERWRITE TABLE dest2 SELECT src.key, src.value WHERE src.key >= 100 and src.key < 200 -INSERT OVERWRITE TABLE dest3 PARTITION(ds='2008-04-08', hr='12') SELECT src.key WHERE src.key >= 200 +INSERT OVERWRITE TABLE dest3 PARTITION(ds='2008-04-08', hr='12') SELECT src.key, 2 WHERE src.key >= 200 Index: src/contrib/hive/ql/src/test/queries/positive/input3.q =================================================================== --- src/contrib/hive/ql/src/test/queries/positive/input3.q (revision 711472) +++ src/contrib/hive/ql/src/test/queries/positive/input3.q (working copy) @@ -1,5 +1,5 @@ FROM src INSERT OVERWRITE TABLE dest1 SELECT src.* WHERE src.key < 100 INSERT OVERWRITE TABLE dest2 SELECT src.key, src.value WHERE src.key >= 100 and src.key < 200 -INSERT OVERWRITE TABLE dest3 PARTITION(ds='2008-04-08', hr='12') SELECT src.key WHERE src.key >= 200 and src.key < 300 +INSERT OVERWRITE TABLE dest3 PARTITION(ds='2008-04-08', hr='12') SELECT src.key, 2 WHERE src.key >= 200 and src.key < 300 INSERT OVERWRITE DIRECTORY '../../../../build/contrib/hive/ql/test/data/warehouse/dest4.out' SELECT src.value WHERE src.key >= 300 Index: src/contrib/hive/ql/src/test/queries/positive/join4.q =================================================================== --- src/contrib/hive/ql/src/test/queries/positive/join4.q (revision 711472) +++ src/contrib/hive/ql/src/test/queries/positive/join4.q (working copy) @@ -10,5 +10,5 @@ ON (a.c1 = b.c3) SELECT a.c1 AS c1, a.c2 AS c2, b.c3 AS c3, b.c4 AS c4 ) c -INSERT OVERWRITE TABLE dest1 SELECT c.c1, c.c2, c.c3, c.c4 +SELECT c.c1, c.c2, c.c3, c.c4 Index: src/contrib/hive/ql/src/test/queries/positive/join5.q =================================================================== --- src/contrib/hive/ql/src/test/queries/positive/join5.q (revision 711472) +++ src/contrib/hive/ql/src/test/queries/positive/join5.q (working copy) @@ -10,6 +10,6 @@ ON (a.c1 = b.c3) SELECT a.c1 AS c1, a.c2 AS c2, b.c3 AS c3, b.c4 AS c4 ) c -INSERT OVERWRITE TABLE dest1 SELECT c.c1, c.c2, c.c3, c.c4 +SELECT c.c1, c.c2, c.c3, c.c4 Index: src/contrib/hive/ql/src/test/queries/positive/input_testxpath2.q =================================================================== --- src/contrib/hive/ql/src/test/queries/positive/input_testxpath2.q (revision 711472) +++ src/contrib/hive/ql/src/test/queries/positive/input_testxpath2.q (working copy) @@ -1,2 +1,2 @@ FROM src_thrift -INSERT OVERWRITE TABLE dest1 SELECT size(src_thrift.lint), size(src_thrift.lintstring), size(src_thrift.mstringstring) where src_thrift.lint IS NOT NULL AND NOT (src_thrift.mstringstring IS NULL) +SELECT size(src_thrift.lint), size(src_thrift.lintstring), size(src_thrift.mstringstring) where src_thrift.lint IS NOT NULL AND NOT (src_thrift.mstringstring IS NULL) Index: src/contrib/hive/ql/src/test/queries/positive/join6.q =================================================================== --- src/contrib/hive/ql/src/test/queries/positive/join6.q (revision 711472) +++ src/contrib/hive/ql/src/test/queries/positive/join6.q (working copy) @@ -10,7 +10,7 @@ ON (a.c1 = b.c3) SELECT a.c1 AS c1, a.c2 AS c2, b.c3 AS c3, b.c4 AS c4 ) c -INSERT OVERWRITE TABLE dest1 SELECT c.c1, c.c2, c.c3, c.c4 +SELECT c.c1, c.c2, c.c3, c.c4 Index: src/contrib/hive/ql/src/test/queries/positive/join7.q =================================================================== --- src/contrib/hive/ql/src/test/queries/positive/join7.q (revision 711472) +++ src/contrib/hive/ql/src/test/queries/positive/join7.q (working copy) @@ -15,7 +15,7 @@ ON (a.c1 = c.c5) SELECT a.c1 AS c1, a.c2 AS c2, b.c3 AS c3, b.c4 AS c4, c.c5 AS c5, c.c6 AS c6 ) c -INSERT OVERWRITE TABLE dest1 SELECT c.c1, c.c2, c.c3, c.c4, c.c5, c.c6 +SELECT c.c1, c.c2, c.c3, c.c4, c.c5, c.c6 Index: src/contrib/hive/ql/src/test/queries/positive/input8.q =================================================================== --- src/contrib/hive/ql/src/test/queries/positive/input8.q (revision 711472) +++ src/contrib/hive/ql/src/test/queries/positive/input8.q (working copy) @@ -1,2 +1,2 @@ FROM src1 -INSERT OVERWRITE TABLE dest1 SELECT 4 + NULL, src1.key - NULL, NULL + NULL +SELECT 4 + NULL, src1.key - NULL, NULL + NULL Index: src/contrib/hive/ql/src/test/queries/positive/join8.q =================================================================== --- src/contrib/hive/ql/src/test/queries/positive/join8.q (revision 711472) +++ src/contrib/hive/ql/src/test/queries/positive/join8.q (working copy) @@ -10,5 +10,5 @@ ON (a.c1 = b.c3) SELECT a.c1 AS c1, a.c2 AS c2, b.c3 AS c3, b.c4 AS c4 ) c -INSERT OVERWRITE TABLE dest1 SELECT c.c1, c.c2, c.c3, c.c4 where c.c3 IS NULL AND c.c1 IS NOT NULL +SELECT c.c1, c.c2, c.c3, c.c4 where c.c3 IS NULL AND c.c1 IS NOT NULL Index: src/contrib/hive/ql/src/test/queries/positive/udf1.q =================================================================== --- src/contrib/hive/ql/src/test/queries/positive/udf1.q (revision 711472) +++ src/contrib/hive/ql/src/test/queries/positive/udf1.q (working copy) @@ -1,4 +1,4 @@ -FROM src INSERT OVERWRITE TABLE dest1 SELECT 'a' LIKE '%a%', 'b' LIKE '%a%', 'ab' LIKE '%a%', 'ab' LIKE '%a_', +FROM src SELECT 'a' LIKE '%a%', 'b' LIKE '%a%', 'ab' LIKE '%a%', 'ab' LIKE '%a_', '%_' LIKE '\%\_', 'ab' LIKE '\%\_', 'ab' LIKE '_a%', 'ab' LIKE 'a', '' RLIKE '.*', 'a' RLIKE '[ab]', '' RLIKE '[ab]', 'hadoop' RLIKE '[a-z]*', 'hadoop' RLIKE 'o*', REGEXP_REPLACE('abc', 'b', 'c'), REGEXP_REPLACE('abc', 'z', 'a'), REGEXP_REPLACE('abbbb', 'bb', 'b'), REGEXP_REPLACE('hadoop', '(.)[a-z]*', '$1ive') Index: src/contrib/hive/ql/src/test/queries/positive/input_testxpath.q =================================================================== --- src/contrib/hive/ql/src/test/queries/positive/input_testxpath.q (revision 711472) +++ src/contrib/hive/ql/src/test/queries/positive/input_testxpath.q (working copy) @@ -1,2 +1,2 @@ FROM src_thrift -INSERT OVERWRITE TABLE dest1 SELECT src_thrift.lint[1], src_thrift.lintstring[0].mystring, src_thrift.mstringstring['key_2'] +SELECT src_thrift.lint[1], src_thrift.lintstring[0].mystring, src_thrift.mstringstring['key_2'] Index: src/contrib/hive/ql/src/test/queries/positive/input_part1.q =================================================================== --- src/contrib/hive/ql/src/test/queries/positive/input_part1.q (revision 711472) +++ src/contrib/hive/ql/src/test/queries/positive/input_part1.q (working copy) @@ -1,2 +1,2 @@ FROM srcpart -INSERT OVERWRITE TABLE dest1 SELECT srcpart.key, srcpart.value, srcpart.hr, srcpart.ds WHERE srcpart.key < 100 and srcpart.ds = '2008-04-08' and srcpart.hr = '12' +SELECT srcpart.key, srcpart.value, srcpart.hr, srcpart.ds WHERE srcpart.key < 100 and srcpart.ds = '2008-04-08' and srcpart.hr = '12' Index: src/contrib/hive/ql/src/test/queries/positive/groupby2.q =================================================================== --- src/contrib/hive/ql/src/test/queries/positive/groupby2.q (revision 711472) +++ src/contrib/hive/ql/src/test/queries/positive/groupby2.q (working copy) @@ -1,2 +1,2 @@ FROM src -INSERT OVERWRITE TABLE dest1 SELECT substr(src.key,0,1), count(DISTINCT substr(src.value,4)), concat(substr(src.key,0,1),sum(substr(src.value,4))) GROUP BY substr(src.key,0,1) +SELECT substr(src.key,0,1), count(DISTINCT substr(src.value,4)), concat(substr(src.key,0,1),sum(substr(src.value,4))) GROUP BY substr(src.key,0,1) Index: src/contrib/hive/ql/src/test/queries/positive/groupby3.q =================================================================== --- src/contrib/hive/ql/src/test/queries/positive/groupby3.q (revision 711472) +++ src/contrib/hive/ql/src/test/queries/positive/groupby3.q (working copy) @@ -1,2 +1,2 @@ FROM src -INSERT OVERWRITE TABLE dest1 SELECT sum(substr(src.value,4)), avg(substr(src.value,4)), avg(DISTINCT substr(src.value,4)), max(substr(src.value,4)), min(substr(src.value,4)) +SELECT sum(substr(src.value,4)), avg(substr(src.value,4)), avg(DISTINCT substr(src.value,4)), max(substr(src.value,4)), min(substr(src.value,4)) Index: src/contrib/hive/ql/src/test/queries/positive/groupby4.q =================================================================== --- src/contrib/hive/ql/src/test/queries/positive/groupby4.q (revision 711472) +++ src/contrib/hive/ql/src/test/queries/positive/groupby4.q (working copy) @@ -1,2 +1,2 @@ FROM src -INSERT OVERWRITE TABLE dest1 SELECT substr(src.key,0,1) GROUP BY substr(src.key,0,1) +SELECT substr(src.key,0,1) GROUP BY substr(src.key,0,1) Index: src/contrib/hive/ql/src/test/queries/positive/groupby5.q =================================================================== --- src/contrib/hive/ql/src/test/queries/positive/groupby5.q (revision 711472) +++ src/contrib/hive/ql/src/test/queries/positive/groupby5.q (working copy) @@ -1,4 +1,4 @@ -INSERT OVERWRITE TABLE dest1 + SELECT src.key, sum(substr(src.value,4)) FROM src GROUP BY src.key Index: src/contrib/hive/ql/src/test/queries/positive/groupby6.q =================================================================== --- src/contrib/hive/ql/src/test/queries/positive/groupby6.q (revision 711472) +++ src/contrib/hive/ql/src/test/queries/positive/groupby6.q (working copy) @@ -1,2 +1,2 @@ FROM src -INSERT OVERWRITE TABLE dest1 SELECT DISTINCT substr(src.value,4,1) +SELECT DISTINCT substr(src.value,4,1) Index: src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java =================================================================== --- src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java (revision 711472) +++ src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java (working copy) @@ -23,6 +23,7 @@ import java.util.AbstractMap; import java.util.ArrayList; import java.util.HashMap; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Properties; @@ -40,13 +41,13 @@ import org.apache.hadoop.hive.metastore.api.Order; import org.apache.hadoop.hive.metastore.api.SerDeInfo; import org.apache.hadoop.hive.metastore.api.StorageDescriptor; +import org.apache.hadoop.hive.serde.Constants; +import org.apache.hadoop.hive.serde2.SerDeUtils; import org.apache.hadoop.hive.serde2.Deserializer; import org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe; -import org.apache.hadoop.hive.serde2.SerDe; import org.apache.hadoop.hive.serde2.SerDeException; import org.apache.hadoop.hive.serde2.objectinspector.StructField; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; -import org.apache.hadoop.hive.serde.Constants; import org.apache.hadoop.io.Writable; import org.apache.hadoop.io.WritableComparable; import org.apache.hadoop.mapred.InputFormat; @@ -129,6 +130,14 @@ sd.getSerdeInfo().setParameters(new HashMap()); } + public void reinitSerDe() throws HiveException { + try { + deserializer = MetaStoreUtils.getDeserializer(Hive.get().getConf(), this.getTTable()); + } catch (MetaException e) { + throw new HiveException(e); + } + } + protected void initSerDe() throws HiveException { if (deserializer == null) { try { @@ -138,13 +147,16 @@ } } } - + public void checkValidity() throws HiveException { // check for validity String name = getTTable().getTableName(); if (null == name || name.length() == 0 || !MetaStoreUtils.validateName(name)) { throw new HiveException("[" + name + "]: is not a valid table name"); } + if (0 == getCols().size()) { + throw new HiveException("atleast one column must be specified for the table"); + } if (null == getDeserializer()) { throw new HiveException("must specify a non-null serDe"); } @@ -154,6 +166,30 @@ if (null == getOutputFormatClass()) { throw new HiveException("must specify an OutputFormat class"); } + + Iterator iterCols = getCols().iterator(); + List colNames = new ArrayList(); + while (iterCols.hasNext()) { + String colName = iterCols.next().getName(); + Iterator iter = colNames.iterator(); + while (iter.hasNext()) { + String oldColName = iter.next(); + if (colName.equalsIgnoreCase(oldColName)) + throw new HiveException("Duplicate column name " + colName + " in the table definition."); + } + colNames.add(colName.toLowerCase()); + } + + if (getPartCols() != null) + { + // there is no overlap between columns and partitioning columns + Iterator partColsIter = getPartCols().iterator(); + while (partColsIter.hasNext()) { + String partCol = partColsIter.next().getName(); + if(colNames.contains(partCol.toLowerCase())) + throw new HiveException("Partition collumn name " + partCol + " conflicts with table columns."); + } + } return; } @@ -190,6 +226,13 @@ } final public Deserializer getDeserializer() { + if(deserializer == null) { + try { + initSerDe(); + } catch (HiveException e) { + LOG.error("Error in initializing serde.", e); + } + } return deserializer; } @@ -360,9 +403,30 @@ } public List getCols() { - return getTTable().getSd().getCols(); + boolean isNative = SerDeUtils.isNativeSerDe(getSerializationLib()); + if (isNative) + return getTTable().getSd().getCols(); + else { + try { + return Hive.getFieldsFromDeserializer(getName(), getDeserializer()); + } catch (HiveException e) { + LOG.error("Unable to get field from serde: " + getSerializationLib(), e); + } + return new ArrayList(); + } } + /** + * Returns a list of all the columns of the table (data columns + partition columns in that order. + * + * @return List + */ + public List getAllCols() { + ArrayList f_list = new ArrayList(); + f_list.addAll(getPartCols()); + f_list.addAll(getCols()); + return f_list; + } public void setPartCols(List partCols) { getTTable().setPartitionKeys(partCols); } Index: src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/metadata/Partition.java =================================================================== --- src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/metadata/Partition.java (revision 711472) +++ src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/metadata/Partition.java (working copy) @@ -36,7 +36,6 @@ import org.apache.hadoop.hive.metastore.Warehouse; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.MetaException; -import org.apache.hadoop.hive.metastore.api.Order; import org.apache.hadoop.hive.metastore.api.StorageDescriptor; /** @@ -205,6 +204,10 @@ return(ret); } + public Path getPartitionPath() { + return this.partPath; + } + final public URI getDataLocation() { return this.partURI; } Index: src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java =================================================================== --- src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java (revision 711472) +++ src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java (working copy) @@ -38,13 +38,14 @@ import org.apache.hadoop.hive.metastore.MetaStoreClient; import org.apache.hadoop.hive.metastore.MetaStoreUtils; import org.apache.hadoop.hive.metastore.api.AlreadyExistsException; -import org.apache.hadoop.hive.metastore.api.Constants; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.InvalidOperationException; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; import org.apache.hadoop.hive.metastore.api.UnknownTableException; import org.apache.hadoop.hive.ql.parse.ParseDriver; +import org.apache.hadoop.hive.serde2.Deserializer; +import org.apache.hadoop.hive.serde2.SerDeException; import org.apache.hadoop.io.Writable; import org.apache.hadoop.io.WritableComparable; import org.apache.hadoop.mapred.InputFormat; @@ -170,6 +171,7 @@ } tbl.setSerializationLib(org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe.class.getName()); tbl.setNumBuckets(bucketCount); + tbl.setBucketCols(bucketCols); createTable(tbl); } @@ -196,6 +198,9 @@ public void createTable(Table tbl) throws HiveException { try { tbl.initSerDe(); + if(tbl.getCols().size() == 0) { + tbl.setFields(MetaStoreUtils.getFieldsFromDeserializer(tbl.getName(), tbl.getDeserializer())); + } tbl.checkValidity(); msc.createTable(tbl.getTTable()); } catch (Exception e) { @@ -652,5 +657,16 @@ } return new MetaStoreClient(this.conf); } + + public static List getFieldsFromDeserializer(String name, Deserializer serde) throws HiveException { + try { + return MetaStoreUtils.getFieldsFromDeserializer(name, serde); + } catch (SerDeException e) { + throw new HiveException("Error in getting fields from serde.", e); + } catch (MetaException e) { + throw new HiveException("Error in getting fields from serde.", e); + } + } + }; Index: src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java =================================================================== --- src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java (revision 711472) +++ src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java (working copy) @@ -26,14 +26,19 @@ import org.apache.log4j.*; import java.net.URL; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.ql.metadata.Hive; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.ql.exec.Utilities; import org.apache.commons.lang.StringUtils; public class SessionState { - + + public static Log LOG = LogFactory.getLog("SessionState"); + public static LogHelper console = new LogHelper(LOG); + /** * current configuration */ @@ -146,11 +151,13 @@ public static SessionState start(HiveConf conf) { ss = new SessionState (conf); ss.getConf().setVar(HiveConf.ConfVars.HIVESESSIONID, makeSessionId()); + console = new LogHelper(LOG); return (ss); } public static SessionState start(SessionState startSs) { ss = startSs; + console = new LogHelper(LOG); ss.getConf().setVar(HiveConf.ConfVars.HIVESESSIONID, makeSessionId()); return ss; } @@ -162,6 +169,10 @@ return ss; } + public static LogHelper getConsole() { + return console; + } + private static String makeSessionId() { GregorianCalendar gc = new GregorianCalendar(); String userid = System.getProperty("user.name"); @@ -242,4 +253,107 @@ LOG.error(error + StringUtils.defaultString(detail)); } } + + public static String validateFile(Set curFiles, String newFile) { + SessionState ss = SessionState.get(); + LogHelper console = SessionState.getConsole(); + Configuration conf = (ss == null) ? new Configuration() : ss.getConf(); + + try { + if(Utilities.realFile(newFile, conf) != null) + return newFile; + else { + console.printError(newFile + " does not exist"); + return null; + } + } catch (IOException e) { + console.printError("Unable to validate " + newFile + "\nException: " + e.getMessage(), + "\n" + org.apache.hadoop.util.StringUtils.stringifyException(e)); + return null; + } + } + + public static interface ResourceHook { + public String preHook(Set cur, String s); + } + + public static enum ResourceType { + FILE(new ResourceHook () { + public String preHook(Set cur, String s) { return validateFile(cur, s); } + }); + + public ResourceHook hook; + + ResourceType(ResourceHook hook) { + this.hook = hook; + } + }; + + public static ResourceType find_resource_type(String s) { + + s = s.trim().toUpperCase(); + + try { + return ResourceType.valueOf(s); + } catch (IllegalArgumentException e) { + } + + // try singular + if(s.endsWith("S")) { + s = s.substring(0, s.length()-1); + } else { + return null; + } + + try { + return ResourceType.valueOf(s); + } catch (IllegalArgumentException e) { + } + return null; + } + + private HashMap> resource_map = new HashMap> (); + + public void add_resource(ResourceType t, String value) { + if(resource_map.get(t) == null) { + resource_map.put(t, new HashSet ()); + } + + String fnlVal = value; + if(t.hook != null) { + fnlVal = t.hook.preHook(resource_map.get(t), value); + if(fnlVal == null) + return; + } + resource_map.get(t).add(fnlVal); + } + + public boolean delete_resource(ResourceType t, String value) { + if(resource_map.get(t) == null) { + return false; + } + return (resource_map.get(t).remove(value)); + } + + public Set list_resource(ResourceType t, List filter) { + if(resource_map.get(t) == null) { + return null; + } + Set orig = resource_map.get(t); + if(filter == null) { + return orig; + } else { + Set fnl = new HashSet (); + for(String one: orig) { + if(filter.contains(one)) { + fnl.add(one); + } + } + return fnl; + } + } + + public void delete_resource(ResourceType t) { + resource_map.remove (t); + } } Index: src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/exec/GroupByOperator.java =================================================================== --- src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/exec/GroupByOperator.java (revision 711472) +++ src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/exec/GroupByOperator.java (working copy) @@ -18,9 +18,12 @@ package org.apache.hadoop.hive.ql.exec; +import java.util.Arrays; import java.util.HashMap; import java.util.ArrayList; import java.util.List; +import java.util.Iterator; +import java.util.Map; import java.io.Serializable; import java.lang.reflect.Method; @@ -61,98 +64,112 @@ transient protected HashMap, UDAF[]> hashAggregations; transient boolean firstRow; - + transient long totalMemory; + transient boolean hashAggr; + public void initialize(Configuration hconf) throws HiveException { super.initialize(hconf); - try { - // init keyFields - keyFields = new ExprNodeEvaluator[conf.getKeys().size()]; - for (int i = 0; i < keyFields.length; i++) { - keyFields[i] = ExprNodeEvaluatorFactory.get(conf.getKeys().get(i)); + totalMemory = Runtime.getRuntime().totalMemory(); + + // init keyFields + keyFields = new ExprNodeEvaluator[conf.getKeys().size()]; + for (int i = 0; i < keyFields.length; i++) { + keyFields[i] = ExprNodeEvaluatorFactory.get(conf.getKeys().get(i)); + } + + // init aggregationParameterFields + aggregationParameterFields = new ExprNodeEvaluator[conf.getAggregators().size()][]; + for (int i = 0; i < aggregationParameterFields.length; i++) { + ArrayList parameters = conf.getAggregators().get(i).getParameters(); + aggregationParameterFields[i] = new ExprNodeEvaluator[parameters.size()]; + for (int j = 0; j < parameters.size(); j++) { + aggregationParameterFields[i][j] = ExprNodeEvaluatorFactory.get(parameters.get(j)); } - - // init aggregationParameterFields - aggregationParameterFields = new ExprNodeEvaluator[conf.getAggregators().size()][]; - for (int i = 0; i < aggregationParameterFields.length; i++) { - ArrayList parameters = conf.getAggregators().get(i).getParameters(); - aggregationParameterFields[i] = new ExprNodeEvaluator[parameters.size()]; - for (int j = 0; j < parameters.size(); j++) { - aggregationParameterFields[i][j] = ExprNodeEvaluatorFactory.get(parameters.get(j)); + } + // init aggregationIsDistinct + aggregationIsDistinct = new boolean[conf.getAggregators().size()]; + for(int i=0; i[]) new Class[conf.getAggregators().size()]; + for (int i = 0; i < conf.getAggregators().size(); i++) { + aggregationDesc agg = conf.getAggregators().get(i); + aggregationClasses[i] = agg.getAggregationClass(); + } + + // init aggregations, aggregationsAggregateMethods, + // aggregationsEvaluateMethods + aggregationsAggregateMethods = new Method[aggregationClasses.length]; + aggregationsEvaluateMethods = new Method[aggregationClasses.length]; + String evaluateMethodName = ((conf.getMode() == groupByDesc.Mode.PARTIAL1 || conf.getMode() == groupByDesc.Mode.HASH || + conf.getMode() == groupByDesc.Mode.PARTIAL2) + ? "evaluatePartial" : "evaluate"); + + for(int i=0; i[]) new Class[conf.getAggregators().size()]; - for (int i = 0; i < conf.getAggregators().size(); i++) { - aggregationDesc agg = conf.getAggregators().get(i); - aggregationClasses[i] = agg.getAggregationClass(); - } - - // init aggregations, aggregationsAggregateMethods, // aggregationsEvaluateMethods - aggregationsAggregateMethods = new Method[aggregationClasses.length]; - aggregationsEvaluateMethods = new Method[aggregationClasses.length]; - String aggregateMethodName = (conf.getMode() == groupByDesc.Mode.PARTIAL2 - ? "aggregatePartial" : "aggregate"); - String evaluateMethodName = ((conf.getMode() == groupByDesc.Mode.PARTIAL1 || conf.getMode() == groupByDesc.Mode.HASH) - ? "evaluatePartial" : "evaluate"); - for(int i=0; i, UDAF[]>(); + if (null == aggregationsEvaluateMethods[i]) { + throw new HiveException("Cannot find " + evaluateMethodName + " method of UDAF class " + + aggregationClasses[i].getName() + "!"); } - // init objectInspectors - int totalFields = keyFields.length + aggregationClasses.length; - objectInspectors = new ArrayList(totalFields); - for(int i=0; i, UDAF[]>(); + hashAggr = true; + } + // init objectInspectors + int totalFields = keyFields.length + aggregationClasses.length; + objectInspectors = new ArrayList(totalFields); + for(int i=0; i newKeys) throws HiveException { + // Prepare aggs for updating + UDAF[] aggs = null; + boolean newEntry = false; + + // hash-based aggregations + aggs = hashAggregations.get(newKeys); + if (aggs == null) { + aggs = newAggregations(); + hashAggregations.put(newKeys, aggs); + newEntry = true; + } + + // Update the aggs + updateAggregations(aggs, row, rowInspector, true, newEntry, null); + + // currently, we use a simple approximation - if 90% of memory is being + // used, flush + long freeMemory = Runtime.getRuntime().freeMemory(); + if (shouldBeFlushed(totalMemory, freeMemory)) { + flush(); + } + } + + private void processAggr(Object row, ObjectInspector rowInspector, ArrayList newKeys) throws HiveException { + // Prepare aggs for updating + UDAF[] aggs = null; + Object[][] lastInvoke = null; + boolean keysAreEqual = newKeys.equals(currentKeys); + + // forward the current keys if needed for sort-based aggregation + if (currentKeys != null && !keysAreEqual) + forward(currentKeys, aggregations); + + // Need to update the keys? + if (currentKeys == null || !keysAreEqual) { + currentKeys = newKeys; + + // init aggregations + for(UDAF aggregation: aggregations) + aggregation.init(); + + // clear parameters in last-invoke + for(int i=0; i= total) + return true; + return false; + } + + private void flush() throws HiveException { + // Currently, the algorithm flushes 10% of the entries - this can be + // changed in the future + + int oldSize = hashAggregations.size(); + Iterator iter = hashAggregations.entrySet().iterator(); + int numDel = 0; + while (iter.hasNext()) { + Map.Entry, UDAF[]> m = (Map.Entry)iter.next(); + forward(m.getKey(), m.getValue()); + iter.remove(); + numDel++; + if (numDel * 10 >= oldSize) + return; + } + } + /** * Forward a record of keys and aggregation results. * @@ -262,14 +336,19 @@ * The keys in the record * @throws HiveException */ - protected void forward(ArrayList keys, UDAF[] aggs) throws Exception { + protected void forward(ArrayList keys, UDAF[] aggs) throws HiveException { int totalFields = keys.size() + aggs.length; List a = new ArrayList(totalFields); for(int i=0; i implements Serializable { - // a list of value expressions for each alias are maintained + static final private Log LOG = LogFactory.getLog(JoinOperator.class.getName()); + + // a list of value expressions for each alias are maintained public static class JoinExprMap { ExprNodeEvaluator[] valueFields; @@ -56,62 +61,79 @@ } - public static class IntermediateObject{ + public static class IntermediateObject { ArrayList[] objs; int curSize; public IntermediateObject(ArrayList[] objs, int curSize) { - this.objs = objs; + this.objs = objs; this.curSize = curSize; } - public ArrayList[] getObjs() { return objs; } - public int getCurSize() { return curSize; } - public void pushObj(ArrayList obj) { objs[curSize++] = obj; } - public void popObj() { curSize--; } + public ArrayList[] getObjs() { + return objs; + } + + public int getCurSize() { + return curSize; + } + + public void pushObj(ArrayList obj) { + objs[curSize++] = obj; + } + + public void popObj() { + curSize--; + } } transient protected int numValues; // number of aliases transient static protected ExprNodeEvaluator aliasField; + transient static protected ExprNodeEvaluator keyField; transient protected HashMap joinExprs; - transient static protected Byte[] order; // order in which the results should be outputted + transient static protected Byte[] order; // order in which the results should + // be outputted transient protected joinCond[] condn; transient protected boolean noOuterJoin; - transient private Object[] dummyObj; // for outer joins, contains the potential nulls for the concerned aliases + transient private Object[] dummyObj; // for outer joins, contains the + // potential nulls for the concerned + // aliases transient private Vector>[] dummyObjVectors; transient private Stack>> iterators; transient private int totalSz; // total size of the composite object transient ObjectInspector joinOutputObjectInspector; - - static - { - aliasField = ExprNodeEvaluatorFactory.get(new exprNodeColumnDesc(String.class, Utilities.ReduceField.ALIAS.toString())); + + static { + aliasField = ExprNodeEvaluatorFactory.get(new exprNodeColumnDesc( + String.class, Utilities.ReduceField.ALIAS.toString())); + keyField = ExprNodeEvaluatorFactory.get(new exprNodeColumnDesc( + String.class, Utilities.ReduceField.KEY.toString())); } + + HashMap>> storage; + int joinEmitInterval = -1; - HashMap>> storage; - public void initialize(Configuration hconf) throws HiveException { super.initialize(hconf); totalSz = 0; // Map that contains the rows for each alias storage = new HashMap>>(); - + numValues = conf.getExprs().size(); joinExprs = new HashMap(); - if (order == null) - { + if (order == null) { order = new Byte[numValues]; for (int i = 0; i < numValues; i++) - order[i] = (byte)i; + order[i] = (byte) i; } condn = conf.getConds(); noOuterJoin = conf.getNoOuterJoin(); Map> map = conf.getExprs(); Iterator entryIter = map.entrySet().iterator(); while (entryIter.hasNext()) { - Map.Entry e = (Map.Entry)entryIter.next(); - Byte key = (Byte)e.getKey(); - ArrayList expr = (ArrayList)e.getValue(); + Map.Entry e = (Map.Entry) entryIter.next(); + Byte key = (Byte) e.getKey(); + ArrayList expr = (ArrayList) e.getValue(); int sz = expr.size(); totalSz += sz; @@ -123,12 +145,15 @@ joinExprs.put(key, new JoinExprMap(valueFields)); } - ArrayList structFieldObjectInspectors = new ArrayList(totalSz); - for(int i=0; i structFieldObjectInspectors = new ArrayList( + totalSz); + for (int i = 0; i < totalSz; i++) { + structFieldObjectInspectors.add(ObjectInspectorFactory + .getStandardPrimitiveObjectInspector(String.class)); } - joinOutputObjectInspector = ObjectInspectorFactory.getStandardStructObjectInspector( - ObjectInspectorUtils.getIntegerArray(totalSz), structFieldObjectInspectors); + joinOutputObjectInspector = ObjectInspectorFactory + .getStandardStructObjectInspector(ObjectInspectorUtils + .getIntegerArray(totalSz), structFieldObjectInspectors); dummyObj = new Object[numValues]; dummyObjVectors = new Vector[numValues]; @@ -149,6 +174,8 @@ } iterators = new Stack>>(); + + joinEmitInterval = HiveConf.getIntVar(hconf, HiveConf.ConfVars.HIVEPARTITIONNAME); } public void startGroup() throws HiveException { @@ -159,7 +186,9 @@ } InspectableObject tempAliasInspectableObject = new InspectableObject(); - public void process(Object row, ObjectInspector rowInspector) throws HiveException { + + public void process(Object row, ObjectInspector rowInspector) + throws HiveException { try { // get alias aliasField.evaluate(row, rowInspector, tempAliasInspectableObject); @@ -176,15 +205,40 @@ nr.add(tempAliasInspectableObject.o); } + // Are we consuming too much memory + if (storage.get(alias).size() == joinEmitInterval) { + if (alias == numValues - 1) { + // The input is sorted by alias, so if we are already in the last join + // operand, + // we can emit some results now. + // Note this has to be done before adding the current row to the + // storage, + // to preserve the correctness for outer joins. + checkAndGenObject(); + storage.get(alias).clear(); + } else { + // Output a warning if we reached at least 1000 rows for a join + // operand + // We won't output a warning for the last join operand since the size + // will never goes to joinEmitInterval. + InspectableObject io = new InspectableObject(); + keyField.evaluate(row, rowInspector, io); + LOG.warn("table " + alias + + " has more than joinEmitInterval rows for join key " + io.o); + } + } + // Add the value to the vector storage.get(alias).add(nr); + } catch (Exception e) { e.printStackTrace(); throw new HiveException(e); } } - private void createForwardJoinObject(IntermediateObject intObj, boolean[] nullsArr) throws HiveException { + private void createForwardJoinObject(IntermediateObject intObj, + boolean[] nullsArr) throws HiveException { ArrayList nr = new ArrayList(totalSz); for (int i = 0; i < numValues; i++) { Byte alias = order[i]; @@ -204,15 +258,17 @@ } private void copyOldArray(boolean[] src, boolean[] dest) { - for (int i = 0; i < src.length; i++) dest[i] = src[i]; + for (int i = 0; i < src.length; i++) + dest[i] = src[i]; } - private Vector joinObjectsInnerJoin(Vector resNulls, Vector inputNulls, ArrayList newObj, IntermediateObject intObj, int left, boolean newObjNull) - { - if (newObjNull) return resNulls; + private Vector joinObjectsInnerJoin(Vector resNulls, + Vector inputNulls, ArrayList newObj, + IntermediateObject intObj, int left, boolean newObjNull) { + if (newObjNull) + return resNulls; Iterator nullsIter = inputNulls.iterator(); - while (nullsIter.hasNext()) - { + while (nullsIter.hasNext()) { boolean[] oldNulls = nullsIter.next(); boolean oldObjNull = oldNulls[left]; if (!oldObjNull) { @@ -224,12 +280,13 @@ } return resNulls; } - - private Vector joinObjectsLeftOuterJoin(Vector resNulls, Vector inputNulls, ArrayList newObj, IntermediateObject intObj, int left, boolean newObjNull) - { + + private Vector joinObjectsLeftOuterJoin( + Vector resNulls, Vector inputNulls, + ArrayList newObj, IntermediateObject intObj, int left, + boolean newObjNull) { Iterator nullsIter = inputNulls.iterator(); - while (nullsIter.hasNext()) - { + while (nullsIter.hasNext()) { boolean[] oldNulls = nullsIter.next(); boolean oldObjNull = oldNulls[left]; boolean[] newNulls = new boolean[intObj.getCurSize()]; @@ -243,25 +300,25 @@ return resNulls; } - private Vector joinObjectsRightOuterJoin(Vector resNulls, Vector inputNulls, ArrayList newObj, IntermediateObject intObj, int left, boolean newObjNull) - { - if (newObjNull) return resNulls; + private Vector joinObjectsRightOuterJoin( + Vector resNulls, Vector inputNulls, + ArrayList newObj, IntermediateObject intObj, int left, + boolean newObjNull) { + if (newObjNull) + return resNulls; boolean allOldObjsNull = true; Iterator nullsIter = inputNulls.iterator(); - while (nullsIter.hasNext()) - { + while (nullsIter.hasNext()) { boolean[] oldNulls = nullsIter.next(); - if (!oldNulls[left]) - { + if (!oldNulls[left]) { allOldObjsNull = false; break; } } nullsIter = inputNulls.iterator(); - while (nullsIter.hasNext()) - { + while (nullsIter.hasNext()) { boolean[] oldNulls = nullsIter.next(); boolean oldObjNull = oldNulls[left]; @@ -270,8 +327,7 @@ copyOldArray(oldNulls, newNulls); newNulls[oldNulls.length] = newObjNull; resNulls.add(newNulls); - } - else if (allOldObjsNull) { + } else if (allOldObjsNull) { boolean[] newNulls = new boolean[intObj.getCurSize()]; for (int i = 0; i < intObj.getCurSize() - 1; i++) newNulls[i] = true; @@ -282,12 +338,13 @@ return resNulls; } - private Vector joinObjectsFullOuterJoin(Vector resNulls, Vector inputNulls, ArrayList newObj, IntermediateObject intObj, int left, boolean newObjNull) - { + private Vector joinObjectsFullOuterJoin( + Vector resNulls, Vector inputNulls, + ArrayList newObj, IntermediateObject intObj, int left, + boolean newObjNull) { if (newObjNull) { Iterator nullsIter = inputNulls.iterator(); - while (nullsIter.hasNext()) - { + while (nullsIter.hasNext()) { boolean[] oldNulls = nullsIter.next(); boolean[] newNulls = new boolean[intObj.getCurSize()]; copyOldArray(oldNulls, newNulls); @@ -296,15 +353,13 @@ } return resNulls; } - + boolean allOldObjsNull = true; Iterator nullsIter = inputNulls.iterator(); - while (nullsIter.hasNext()) - { + while (nullsIter.hasNext()) { boolean[] oldNulls = nullsIter.next(); - if (!oldNulls[left]) - { + if (!oldNulls[left]) { allOldObjsNull = false; break; } @@ -312,24 +367,21 @@ boolean rhsPreserved = false; nullsIter = inputNulls.iterator(); - while (nullsIter.hasNext()) - { + while (nullsIter.hasNext()) { boolean[] oldNulls = nullsIter.next(); boolean oldObjNull = oldNulls[left]; - if (!oldObjNull) - { + if (!oldObjNull) { boolean[] newNulls = new boolean[intObj.getCurSize()]; copyOldArray(oldNulls, newNulls); newNulls[oldNulls.length] = newObjNull; resNulls.add(newNulls); - } - else if (oldObjNull) { + } else if (oldObjNull) { boolean[] newNulls = new boolean[intObj.getCurSize()]; copyOldArray(oldNulls, newNulls); newNulls[oldNulls.length] = true; resNulls.add(newNulls); - + if (allOldObjsNull && !rhsPreserved) { newNulls = new boolean[intObj.getCurSize()]; for (int i = 0; i < oldNulls.length; i++) @@ -344,35 +396,35 @@ } /* - * The new input is added to the list of existing inputs. Each entry in the - * array of inputNulls denotes the entries in the intermediate object to - * be used. The intermediate object is augmented with the new object, and - * list of nulls is changed appropriately. The list will contain all non-nulls - * for a inner join. The outer joins are processed appropriately. + * The new input is added to the list of existing inputs. Each entry in the + * array of inputNulls denotes the entries in the intermediate object to be + * used. The intermediate object is augmented with the new object, and list of + * nulls is changed appropriately. The list will contain all non-nulls for a + * inner join. The outer joins are processed appropriately. */ - private Vector joinObjects(Vector inputNulls, ArrayList newObj, IntermediateObject intObj, int joinPos) - { + private Vector joinObjects(Vector inputNulls, + ArrayList newObj, IntermediateObject intObj, int joinPos) { Vector resNulls = new Vector(); boolean newObjNull = newObj == dummyObj[joinPos] ? true : false; - if (joinPos == 0) - { - if (newObjNull) return null; + if (joinPos == 0) { + if (newObjNull) + return null; boolean[] nulls = new boolean[1]; nulls[0] = newObjNull; resNulls.add(nulls); return resNulls; } - + int left = condn[joinPos - 1].getLeft(); int type = condn[joinPos - 1].getType(); - + // process all nulls for RIGHT and FULL OUTER JOINS - if (((type == joinDesc.RIGHT_OUTER_JOIN) || (type == joinDesc.FULL_OUTER_JOIN)) - && !newObjNull && (inputNulls == null)) { + if (((type == joinDesc.RIGHT_OUTER_JOIN) || (type == joinDesc.FULL_OUTER_JOIN)) + && !newObjNull && (inputNulls == null)) { boolean[] newNulls = new boolean[intObj.getCurSize()]; for (int i = 0; i < newNulls.length - 1; i++) newNulls[i] = true; - newNulls[newNulls.length-1] = false; + newNulls[newNulls.length - 1] = false; resNulls.add(newNulls); return resNulls; } @@ -380,41 +432,45 @@ if (inputNulls == null) return null; - if (type == joinDesc.INNER_JOIN) - return joinObjectsInnerJoin(resNulls, inputNulls, newObj, intObj, left, newObjNull); - else if (type == joinDesc.LEFT_OUTER_JOIN) - return joinObjectsLeftOuterJoin(resNulls, inputNulls, newObj, intObj, left, newObjNull); - else if (type == joinDesc.RIGHT_OUTER_JOIN) - return joinObjectsRightOuterJoin(resNulls, inputNulls, newObj, intObj, left, newObjNull); + if (type == joinDesc.INNER_JOIN) + return joinObjectsInnerJoin(resNulls, inputNulls, newObj, intObj, left, + newObjNull); + else if (type == joinDesc.LEFT_OUTER_JOIN) + return joinObjectsLeftOuterJoin(resNulls, inputNulls, newObj, intObj, + left, newObjNull); + else if (type == joinDesc.RIGHT_OUTER_JOIN) + return joinObjectsRightOuterJoin(resNulls, inputNulls, newObj, intObj, + left, newObjNull); assert (type == joinDesc.FULL_OUTER_JOIN); - return joinObjectsFullOuterJoin(resNulls, inputNulls, newObj, intObj, left, newObjNull); + return joinObjectsFullOuterJoin(resNulls, inputNulls, newObj, intObj, left, + newObjNull); } - - /* - * genObject is a recursive function. For the inputs, a array of - * bitvectors is maintained (inputNulls) where each entry denotes whether - * the element is to be used or not (whether it is null or not). The size of - * the bitvector is same as the number of inputs under consideration - * currently. When all inputs are accounted for, the output is forwared - * appropriately. + + /* + * genObject is a recursive function. For the inputs, a array of bitvectors is + * maintained (inputNulls) where each entry denotes whether the element is to + * be used or not (whether it is null or not). The size of the bitvector is + * same as the number of inputs under consideration currently. When all inputs + * are accounted for, the output is forwared appropriately. */ - private void genObject(Vector inputNulls, int aliasNum, IntermediateObject intObj) - throws HiveException { + private void genObject(Vector inputNulls, int aliasNum, + IntermediateObject intObj) throws HiveException { if (aliasNum < numValues) { Iterator> aliasRes = storage.get(order[aliasNum]) - .iterator(); + .iterator(); iterators.push(aliasRes); while (aliasRes.hasNext()) { ArrayList newObj = aliasRes.next(); intObj.pushObj(newObj); - Vector newNulls = joinObjects(inputNulls, newObj, intObj, aliasNum); + Vector newNulls = joinObjects(inputNulls, newObj, intObj, + aliasNum); genObject(newNulls, aliasNum + 1, intObj); intObj.popObj(); } iterators.pop(); - } - else { - if (inputNulls == null) return; + } else { + if (inputNulls == null) + return; Iterator nullsIter = inputNulls.iterator(); while (nullsIter.hasNext()) { boolean[] nullsVec = nullsIter.next(); @@ -429,29 +485,27 @@ * @throws HiveException */ public void endGroup() throws HiveException { - try { - LOG.trace("Join Op: endGroup called: numValues=" + numValues); + LOG.trace("Join Op: endGroup called: numValues=" + numValues); + checkAndGenObject(); + } - // does any result need to be emitted - for (int i = 0; i < numValues; i++) { - Byte alias = order[i]; - if (storage.get(alias).iterator().hasNext() == false) { - if (noOuterJoin) { - LOG.trace("No data for alias=" + i); - return; - } else { - storage.put(alias, dummyObjVectors[i]); - } + private void checkAndGenObject() throws HiveException { + // does any result need to be emitted + for (int i = 0; i < numValues; i++) { + Byte alias = order[i]; + if (storage.get(alias).iterator().hasNext() == false) { + if (noOuterJoin) { + LOG.trace("No data for alias=" + i); + return; + } else { + storage.put(alias, dummyObjVectors[i]); } } + } - LOG.trace("calling genObject"); - genObject(null, 0, new IntermediateObject(new ArrayList[numValues], 0)); - LOG.trace("called genObject"); - } catch (Exception e) { - e.printStackTrace(); - throw new HiveException(e); - } + LOG.trace("calling genObject"); + genObject(null, 0, new IntermediateObject(new ArrayList[numValues], 0)); + LOG.trace("called genObject"); } /** @@ -463,5 +517,3 @@ super.close(abort); } } - - Index: src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchTask.java =================================================================== --- src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchTask.java (revision 711472) +++ src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchTask.java (working copy) @@ -20,7 +20,12 @@ import java.io.Serializable; import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; import java.util.Vector; import java.util.Properties; @@ -29,8 +34,11 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.plan.fetchWork; +import org.apache.hadoop.hive.ql.plan.partitionDesc; +import org.apache.hadoop.hive.ql.plan.tableDesc; import org.apache.hadoop.hive.serde2.Deserializer; import org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.io.Writable; import org.apache.hadoop.io.WritableComparable; import org.apache.hadoop.mapred.FileInputFormat; @@ -45,6 +53,8 @@ import org.apache.hadoop.hive.serde.Constants; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; /** * FetchTask implementation @@ -56,42 +66,27 @@ public void initialize (HiveConf conf) { super.initialize(conf); - splitNum = 0; currRecReader = null; try { // Create a file system handle fs = FileSystem.get(conf); - serde = work.getDeserializerClass().newInstance(); - serde.initialize(null, work.getSchema()); job = new JobConf(conf, ExecDriver.class); - Path inputP = work.getSrcDir(); - if(!fs.exists(inputP)) { - empty = true; - return; - } - - empty = true; - FileStatus[] fStats = fs.listStatus(inputP); - for (FileStatus fStat:fStats) { - if (fStat.getLen() > 0) { - empty = false; - break; - } - } - - if (empty) - return; - - FileInputFormat.setInputPaths(job, inputP); - inputFormat = getInputFormatFromCache(work.getInputFormatClass(), job); - inputSplits = inputFormat.getSplits(job, 1); + mSerde = new MetadataTypedColumnsetSerDe(); Properties mSerdeProp = new Properties(); mSerdeProp.put(Constants.SERIALIZATION_FORMAT, "" + Utilities.tabCode); mSerdeProp.put(Constants.SERIALIZATION_NULL_FORMAT, "NULL"); mSerde.initialize(null, mSerdeProp); + + currPath = null; + currTbl = null; + currPart = null; + iterPath = null; + iterPartDesc = null; totalRows = 0; + tblDataDone = false; + rowWithPart = new Object[2]; } catch (Exception e) { // Bail out ungracefully - we should never hit // this here - but would have hit it in SemanticAnalyzer @@ -136,11 +131,116 @@ private Deserializer serde; private MetadataTypedColumnsetSerDe mSerde; private int totalRows; - private boolean empty; + private Iterator iterPath; + private Iterator iterPartDesc; + private Path currPath; + private partitionDesc currPart; + private tableDesc currTbl; + private boolean tblDataDone; + private StructObjectInspector rowObjectInspector; + private Object[] rowWithPart; + + private void setPrtnDesc() throws Exception { + List partNames = new ArrayList(); + List partValues = new ArrayList(); + + String pcols = currPart.getTableDesc().getProperties().getProperty(org.apache.hadoop.hive.metastore.api.Constants.META_TABLE_PARTITION_COLUMNS); + LinkedHashMap partSpec = currPart.getPartSpec(); + + List partObjectInspectors = new ArrayList(); + String[] partKeys = pcols.trim().split("/"); + for(String key: partKeys) { + partNames.add(key); + partValues.add(partSpec.get(key)); + partObjectInspectors.add(ObjectInspectorFactory.getStandardPrimitiveObjectInspector(String.class)); + } + StructObjectInspector partObjectInspector = ObjectInspectorFactory.getStandardStructObjectInspector(partNames, partObjectInspectors); + rowObjectInspector = (StructObjectInspector)serde.getObjectInspector(); + + rowWithPart[1] = partValues; + rowObjectInspector = ObjectInspectorFactory.getUnionStructObjectInspector(Arrays.asList(new StructObjectInspector[]{ + rowObjectInspector, partObjectInspector})); + } + + private void getNextPath() throws Exception { + // first time + if (iterPath == null) { + if (work.getTblDir() != null) { + if (!tblDataDone) { + currPath = work.getTblDir(); + currTbl = work.getTblDesc(); + if (fs.exists(currPath)) + { + FileStatus[] fStats = fs.listStatus(currPath); + for (FileStatus fStat:fStats) { + if (fStat.getLen() > 0) { + tblDataDone = true; + break; + } + } + } + + if (!tblDataDone) currPath = null; + return; + } else { + currTbl = null; + currPath = null; + } + return; + } + else { + iterPath = work.getPartDir().iterator(); + iterPartDesc = work.getPartDesc().iterator(); + } + } + + while (iterPath.hasNext()) { + Path nxt = iterPath.next(); + partitionDesc prt = iterPartDesc.next(); + if (fs.exists(nxt)) + { + FileStatus[] fStats = fs.listStatus(nxt); + for (FileStatus fStat:fStats) { + if (fStat.getLen() > 0) { + currPath = nxt; + currPart = prt; + return; + } + } + } + } + } private RecordReader getRecordReader() throws Exception { - if (splitNum >= inputSplits.length) - return null; + if (currPath == null) { + getNextPath(); + if (currPath == null) + return null; + + FileInputFormat.setInputPaths(job, currPath); + tableDesc tmp = currTbl; + if (tmp == null) + tmp = currPart.getTableDesc(); + inputFormat = getInputFormatFromCache(tmp.getInputFileFormatClass(), job); + inputSplits = inputFormat.getSplits(job, 1); + splitNum = 0; + serde = tmp.getDeserializerClass().newInstance(); + serde.initialize(null, tmp.getProperties()); + LOG.debug("Creating fetchTask with deserializer typeinfo: " + serde.getObjectInspector().getTypeName()); + LOG.debug("deserializer properties: " + tmp.getProperties()); + if (!tblDataDone) + setPrtnDesc(); + } + + if (splitNum >= inputSplits.length) { + if (currRecReader != null) { + currRecReader.close(); + currRecReader = null; + } + currPath = null; + return getRecordReader(); + } + currRecReader = inputFormat.getRecordReader(inputSplits[splitNum++], job, Reporter.NULL); key = currRecReader.createKey(); value = currRecReader.createValue(); @@ -149,16 +249,15 @@ public boolean fetch(Vector res) { try { - if (empty) - return false; - int numRows = 0; int rowsRet = MAX_ROWS; if ((work.getLimit() >= 0) && ((work.getLimit() - totalRows) < rowsRet)) rowsRet = work.getLimit() - totalRows; if (rowsRet <= 0) { - if (currRecReader != null) + if (currRecReader != null) { currRecReader.close(); + currRecReader = null; + } return false; } @@ -174,12 +273,18 @@ } boolean ret = currRecReader.next(key, value); if (ret) { - Object obj = serde.deserialize(value); - res.add(((Text)mSerde.serialize(obj, serde.getObjectInspector())).toString()); + if (tblDataDone) { + Object obj = serde.deserialize(value); + res.add(((Text)mSerde.serialize(obj, serde.getObjectInspector())).toString()); + } else { + rowWithPart[0] = serde.deserialize(value); + res.add(((Text)mSerde.serialize(rowWithPart, rowObjectInspector)).toString()); + } numRows++; } else { currRecReader.close(); + currRecReader = null; currRecReader = getRecordReader(); if (currRecReader == null) { if (numRows == 0) Index: src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/exec/ScriptOperator.java =================================================================== --- src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/exec/ScriptOperator.java (revision 711472) +++ src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/exec/ScriptOperator.java (working copy) @@ -34,6 +34,7 @@ import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.mapred.LineRecordReader.LineReader; import org.apache.hadoop.util.StringUtils; +import org.apache.hadoop.fs.FileUtil; public class ScriptOperator extends Operator implements Serializable { @@ -89,6 +90,77 @@ } } + + /** + * Maps a relative pathname to an absolute pathname using the + * PATH enviroment. + */ + public class PathFinder + { + String pathenv; // a string of pathnames + String pathSep; // the path seperator + String fileSep; // the file seperator in a directory + + /** + * Construct a PathFinder object using the path from + * the specified system environment variable. + */ + public PathFinder(String envpath) + { + pathenv = System.getenv(envpath); + pathSep = System.getProperty("path.separator"); + fileSep = System.getProperty("file.separator"); + } + + /** + * Appends the specified component to the path list + */ + public void prependPathComponent(String str) + { + pathenv = str + pathSep + pathenv; + } + + /** + * Returns the full path name of this file if it is listed in the + * path + */ + public File getAbsolutePath(String filename) + { + if (pathenv == null || pathSep == null || fileSep == null) { + return null; + } + int val = -1; + String classvalue = pathenv + pathSep; + + while (((val = classvalue.indexOf(pathSep)) >= 0) && + classvalue.length() > 0) { + // + // Extract each entry from the pathenv + // + String entry = classvalue.substring(0, val).trim(); + File f = new File(entry); + + try { + if (f.isDirectory()) { + // + // this entry in the pathenv is a directory. + // see if the required file is in this directory + // + f = new File(entry + fileSep + filename); + } + // + // see if the filename matches and we can read it + // + if (f.isFile() && f.canRead()) { + return f; + } + } catch (Exception exp){ } + classvalue = classvalue.substring(val+1).trim(); + } + return null; + } + } + public void initialize(Configuration hconf) throws HiveException { super.initialize(hconf); statsMap.put(Counter.DESERIALIZE_ERRORS, deserialize_error_count); @@ -104,6 +176,20 @@ scriptInputSerializer.initialize(hconf, conf.getScriptInputInfo().getProperties()); String [] cmdArgs = splitArgs(conf.getScriptCmd()); + + String prog = cmdArgs[0]; + File currentDir = new File(".").getAbsoluteFile(); + + if (!new File(prog).isAbsolute()) { + PathFinder finder = new PathFinder("PATH"); + finder.prependPathComponent(currentDir.toString()); + File f = finder.getAbsolutePath(prog); + if (f != null) { + cmdArgs[0] = f.getAbsolutePath(); + } + f = null; + } + String [] wrappedCmdArgs = addWrapper(cmdArgs); LOG.info("Executing " + Arrays.asList(wrappedCmdArgs)); LOG.info("tablename=" + hconf.get(HiveConf.ConfVars.HIVETABLENAME.varname)); Index: src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java =================================================================== --- src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java (revision 711472) +++ src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java (working copy) @@ -34,11 +34,13 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.FileInputFormat; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.plan.mapredWork; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.io.*; import org.apache.hadoop.hive.ql.session.SessionState.LogHelper; +import org.apache.hadoop.hive.ql.session.SessionState; public class ExecDriver extends Task implements Serializable { @@ -54,12 +56,37 @@ super(); } + public static String getRealFiles(Configuration conf) { + // fill in local files to be added to the task environment + SessionState ss = SessionState.get(); + Set files = (ss == null) ? null : ss.list_resource(SessionState.ResourceType.FILE, null); + if(files != null) { + ArrayList realFiles = new ArrayList (files.size()); + for(String one: files) { + try { + realFiles.add(Utilities.realFile(one, conf)); + } catch (IOException e) { + throw new RuntimeException ("Cannot validate file " + one + + "due to exception: " + e.getMessage(), e); + } + } + return StringUtils.join(realFiles, ","); + } else { + return ""; + } + } + + /** * Initialization when invoked from QL */ public void initialize (HiveConf conf) { super.initialize(conf); job = new JobConf(conf, ExecDriver.class); + String realFiles = getRealFiles(job); + if (realFiles != null && realFiles.length() > 0) { + job.set("tmpfiles", realFiles); + } } /** @@ -121,8 +148,7 @@ } } } - } - ); + }); } } @@ -207,6 +233,7 @@ Utilities.setMapRedWork(job, work); + for(String onefile: work.getPathToAliases().keySet()) { LOG.info("Adding input file " + onefile); FileInputFormat.addInputPaths(job, onefile); @@ -217,8 +244,8 @@ FileOutputFormat.setOutputPath(job, new Path(jobScratchDir)); job.setMapperClass(ExecMapper.class); - job.setMapOutputValueClass(Text.class); job.setMapOutputKeyClass(HiveKey.class); + job.setMapOutputValueClass(BytesWritable.class); job.setNumReduceTasks(work.getNumReduceTasks().intValue()); job.setReducerClass(ExecReducer.class); @@ -265,6 +292,10 @@ inferNumReducers(); JobClient jc = new JobClient(job); + + // make this client wait if job trcker is not behaving well. + Throttle.checkJobTracker(job, LOG); + rj = jc.submitJob(job); // add to list of running jobs so in case of abnormal shutdown can kill it. @@ -306,7 +337,8 @@ } private static void printUsage() { - System.out.println("ExecDriver -plan [-jobconf k1=v1 [-jobconf k2=v2] ...]"); + System.out.println("ExecDriver -plan [-jobconf k1=v1 [-jobconf k2=v2] ...] "+ + "[-files [,] ...]"); System.exit(1); } @@ -314,15 +346,19 @@ String planFileName = null; ArrayList jobConfArgs = new ArrayList (); boolean isSilent = false; + String files = null; try{ for(int i=0; ixxx + Pattern dowait = Pattern.compile("", + Pattern.CASE_INSENSITIVE | Pattern.DOTALL | Pattern.MULTILINE); + String[] results = dowait.split(fetchString); + if (results.length != 2) { + throw new IOException("Throttle: Unable to parse response of URL " + url + + ". Get retuned " + fetchString); + } + dowait = Pattern.compile("", + Pattern.CASE_INSENSITIVE | Pattern.DOTALL | Pattern.MULTILINE); + results = dowait.split(results[1]); + if (results.length < 1) { + throw new IOException("Throttle: Unable to parse response of URL " + url + + ". Get retuned " + fetchString); + } + + // if the jobtracker signalled that the threshold is not exceeded, + // then we return immediately. + if (results[0].trim().compareToIgnoreCase("false") == 0) { + return; + } + + // The JobTracker has exceeded its threshold and is doing a GC. + // The client has to wait and retry. + LOG.warn("Job is being throttled because of resource crunch on the " + + "JobTracker. Will retry in " + retry + " seconds.."); + Thread.sleep(retry * 1000L); + } + } catch (Exception e) { + LOG.warn("Job is not being throttled. " + e); + } + } +} Index: src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/exec/ReduceSinkOperator.java =================================================================== --- src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/exec/ReduceSinkOperator.java (revision 711472) +++ src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/exec/ReduceSinkOperator.java (working copy) @@ -34,7 +34,10 @@ import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; +import org.apache.hadoop.io.BytesWritable; import org.apache.hadoop.io.Text; +import org.apache.hadoop.io.Writable; +import org.apache.hadoop.io.WritableComparable; /** * Reduce Sink Operator sends output to the reduce stage @@ -47,6 +50,7 @@ // TODO: we use MetadataTypedColumnsetSerDe for now, till DynamicSerDe is ready transient Serializer keySerializer; + transient boolean keyIsText; transient Serializer valueSerializer; transient int tag; transient byte[] tagByte = new byte[1]; @@ -74,6 +78,7 @@ tableDesc keyTableDesc = conf.getKeySerializeInfo(); keySerializer = (Serializer)keyTableDesc.getDeserializerClass().newInstance(); keySerializer.initialize(null, keyTableDesc.getProperties()); + keyIsText = keySerializer.getSerializedClass().equals(Text.class); tableDesc valueTableDesc = conf.getValueSerializeInfo(); valueSerializer = (Serializer)valueTableDesc.getDeserializerClass().newInstance(); @@ -89,7 +94,7 @@ transient InspectableObject tempInspectableObject = new InspectableObject(); transient HiveKey keyWritable = new HiveKey(); - transient Text valueText; + transient Writable value; transient ObjectInspector keyObjectInspector; transient ObjectInspector valueObjectInspector; @@ -97,62 +102,81 @@ transient ArrayList valueFieldsObjectInspectors = new ArrayList(); public void process(Object row, ObjectInspector rowInspector) throws HiveException { - // TODO: use DynamicSerDe when that is ready try { - // Generate hashCode for the tuple - int keyHashCode = 0; - if (numPartitionFields == -1) { - keyHashCode = (int)(Math.random() * Integer.MAX_VALUE); - } + // Evaluate the keys ArrayList keys = new ArrayList(keyEval.length); for(ExprNodeEvaluator e: keyEval) { e.evaluate(row, rowInspector, tempInspectableObject); keys.add(tempInspectableObject.o); - if (numPartitionFields == keys.size()) { - keyHashCode = keys.hashCode(); - } + // Construct the keyObjectInspector from the first row if (keyObjectInspector == null) { keyFieldsObjectInspectors.add(tempInspectableObject.oi); } } - if (numPartitionFields > keys.size()) { - keyHashCode = keys.hashCode(); - } + // Construct the keyObjectInspector from the first row if (keyObjectInspector == null) { keyObjectInspector = ObjectInspectorFactory.getStandardStructObjectInspector( ObjectInspectorUtils.getIntegerArray(keyFieldsObjectInspectors.size()), keyFieldsObjectInspectors); } - Text key = (Text)keySerializer.serialize(keys, keyObjectInspector); - if (tag == -1) { - keyWritable.set(key.getBytes(), 0, key.getLength()); + // Serialize the keys and append the tag + if (keyIsText) { + Text key = (Text)keySerializer.serialize(keys, keyObjectInspector); + if (tag == -1) { + keyWritable.set(key.getBytes(), 0, key.getLength()); + } else { + int keyLength = key.getLength(); + keyWritable.setSize(keyLength+1); + System.arraycopy(key.getBytes(), 0, keyWritable.get(), 0, keyLength); + keyWritable.get()[keyLength] = tagByte[0]; + } } else { - int keyLength = key.getLength(); - keyWritable.setSize(keyLength+1); - System.arraycopy(key.getBytes(), 0, keyWritable.get(), 0, keyLength); - keyWritable.get()[keyLength] = tagByte[0]; + // Must be BytesWritable + BytesWritable key = (BytesWritable)keySerializer.serialize(keys, keyObjectInspector); + if (tag == -1) { + keyWritable.set(key.get(), 0, key.getSize()); + } else { + int keyLength = key.getSize(); + keyWritable.setSize(keyLength+1); + System.arraycopy(key.get(), 0, keyWritable.get(), 0, keyLength); + keyWritable.get()[keyLength] = tagByte[0]; + } } - keyWritable.setHashCode(keyHashCode); + // Set the HashCode + if (numPartitionFields == -1) { + keyWritable.setHashCode((int)(Math.random() * Integer.MAX_VALUE)); + } else { + int keyHashCode = 0; + for(int i=0; i < keys.size() && i < numPartitionFields; i++) { + keyHashCode = keyHashCode * 31 + keys.get(i).hashCode(); + } + keyWritable.setHashCode(keyHashCode); + } + // Evaluate the value ArrayList values = new ArrayList(valueEval.length); for(ExprNodeEvaluator e: valueEval) { e.evaluate(row, rowInspector, tempInspectableObject); values.add(tempInspectableObject.o); + // Construct the valueObjectInspector from the first row if (valueObjectInspector == null) { valueFieldsObjectInspectors.add(tempInspectableObject.oi); } } + // Construct the valueObjectInspector from the first row if (valueObjectInspector == null) { valueObjectInspector = ObjectInspectorFactory.getStandardStructObjectInspector( ObjectInspectorUtils.getIntegerArray(valueFieldsObjectInspectors.size()), valueFieldsObjectInspectors); } - valueText = (Text)valueSerializer.serialize(values, valueObjectInspector); + // Serialize the value + value = valueSerializer.serialize(values, valueObjectInspector); } catch (SerDeException e) { throw new HiveException(e); } + try { - out.collect(keyWritable, valueText); + out.collect(keyWritable, value); } catch (IOException e) { throw new HiveException (e); } Index: src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/exec/ColumnInfo.java =================================================================== --- src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/exec/ColumnInfo.java (revision 711472) +++ src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/exec/ColumnInfo.java (working copy) @@ -73,4 +73,11 @@ public void setInternalName(String internalName) { this.internalName = internalName; } + + /** + * Returns the string representation of the ColumnInfo. + */ + public String toString() { + return internalName + ": " + type; + } } Index: src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java =================================================================== --- src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java (revision 711472) +++ src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java (working copy) @@ -140,8 +140,7 @@ // Set up distributed cache DistributedCache.createSymlink(job); String uriWithLink = planPath.toUri().toString() + "#HIVE_PLAN"; - URI[] fileURIs = new URI[] {new URI(uriWithLink)}; - DistributedCache.setCacheFiles(fileURIs, job); + DistributedCache.addCacheFile(new URI(uriWithLink), job); // Cache the object in this process too so lookups don't hit the file system synchronized (Utilities.class) { gWork = w; @@ -198,15 +197,13 @@ public static tableDesc defaultTd; static { // by default we expect ^A separated strings + // This tableDesc does not provide column names. We should always use + // PlanUtils.getDefaultTableDesc(String separatorCode, String columns) + // or getBinarySortableTableDesc(List fieldSchemas) when + // we know the column names. defaultTd = PlanUtils.getDefaultTableDesc("" + Utilities.ctrlaCode); } - public static tableDesc defaultTabTd; - static { - // Default tab-separated tableDesc - defaultTabTd = PlanUtils.getDefaultTableDesc("" + Utilities.tabCode); - } - public final static int newLineCode = 10; public final static int tabCode = 9; public final static int ctrlaCode = 1; @@ -431,4 +428,29 @@ keyClass, valClass, compressionType, codec)); } + + /** + * Shamelessly cloned from GenericOptionsParser + */ + public static String realFile(String newFile, Configuration conf) throws IOException { + Path path = new Path(newFile); + URI pathURI = path.toUri(); + FileSystem fs; + + if (pathURI.getScheme() == null) { + fs = FileSystem.getLocal(conf); + } else { + fs = path.getFileSystem(conf); + } + + if (!fs.exists(path)) { + return null; + } + + try { + fs.close(); + } catch(IOException e){}; + + return (path.makeQualified(fs).toString()); + } } Index: src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java =================================================================== --- src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java (revision 711472) +++ src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java (working copy) @@ -22,6 +22,7 @@ import org.apache.commons.logging.LogFactory; import java.lang.reflect.Method; +import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.List; @@ -29,6 +30,8 @@ import java.lang.Void; import org.apache.hadoop.hive.ql.exec.FunctionInfo.OperatorType; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.plan.groupByDesc; import org.apache.hadoop.hive.ql.udf.*; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; @@ -247,7 +250,7 @@ /** * This method is shared between UDFRegistry and UDAFRegistry. - * methodName will be "evaluate" for UDFRegistry, and "aggregate" for UDAFRegistry. + * methodName will be "evaluate" for UDFRegistry, and "aggregate"/"evaluate"/"evaluatePartial" for UDAFRegistry. */ public static Method getMethodInternal(Class udfClass, String methodName, boolean exact, List> argumentClasses) { int leastImplicitConversions = Integer.MAX_VALUE; @@ -319,6 +322,9 @@ return result; } + /** + * Returns the "aggregate" method of the UDAF. + */ public static Method getUDAFMethod(String name, List> argumentClasses) { Class udaf = getUDAF(name); if (udaf == null) @@ -327,7 +333,62 @@ argumentClasses); } + /** + * Returns the evaluate method for the UDAF based on the aggregation mode. + * See groupByDesc.Mode for details. + * + * @param name name of the UDAF + * @param mode the mode of the aggregation + * @return null if no such UDAF is found + */ + public static Method getUDAFEvaluateMethod(String name, groupByDesc.Mode mode) { + Class udaf = getUDAF(name); + if (udaf == null) + return null; + return FunctionRegistry.getMethodInternal(udaf, + (mode == groupByDesc.Mode.COMPLETE || mode == groupByDesc.Mode.FINAL) + ? "evaluate" : "evaluatePartial", true, + new ArrayList>() ); + } + + /** + * Returns the "aggregate" method of the UDAF. + */ public static Method getUDAFMethod(String name, Class... argumentClasses) { return getUDAFMethod(name, Arrays.asList(argumentClasses)); } + + public static Object invoke(Method m, Object thisObject, Object[] arguments) throws HiveException { + Object o; + try { + o = m.invoke(thisObject, arguments); + } catch (Exception e) { + String thisObjectString = "" + thisObject + " of class " + + (thisObject == null? "null" : thisObject.getClass().getName()); + + StringBuilder argumentString = new StringBuilder(); + if (arguments == null) { + argumentString.append("null"); + } else { + argumentString.append("{"); + for (int i=0; i0) { + argumentString.append(", "); + } + if (arguments[i] == null) { + argumentString.append("null"); + } else { + argumentString.append("" + arguments[i] + ":" + arguments[i].getClass().getName()); + } + } + argumentString.append("} of size " + arguments.length); + } + + throw new HiveException("Unable to execute method " + m + " " + + " on object " + thisObjectString + + " with arguments " + argumentString.toString() + + ":" + e.getMessage()); + } + return o; + } } Index: src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeFuncEvaluator.java =================================================================== --- src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeFuncEvaluator.java (revision 711472) +++ src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeFuncEvaluator.java (working copy) @@ -72,19 +72,8 @@ paramEvaluators[i].evaluate(row, rowInspector, paramInspectableObjects[i]); paramValues[i] = paramInspectableObjects[i].o; } - try { - result.o = udfMethod.invoke(udf, paramValues); - result.oi = outputObjectInspector; - } catch (Exception e) { - if (e instanceof HiveException) { - throw (HiveException)e; - } else if (e instanceof RuntimeException) { - throw (RuntimeException)e; - } else { - throw new HiveException("Unable to execute UDF function " + udf.getClass() + " " - + udfMethod + " on inputs " + "(" + paramValues.length + ") " + Arrays.asList(paramValues) + ": " + e.getMessage(), e); - } - } + result.o = FunctionRegistry.invoke(udfMethod, udf, paramValues); + result.oi = outputObjectInspector; } public ObjectInspector evaluateInspector(ObjectInspector rowInspector) Index: src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java =================================================================== --- src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java (revision 711472) +++ src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java (working copy) @@ -74,6 +74,7 @@ transient HiveConf conf; static final private int separator = Utilities.tabCode; + static final private int singleQuote = '\''; static final private int terminator = Utilities.newLineCode; public void initialize(HiveConf conf) { @@ -95,7 +96,6 @@ // create the table Table tbl = new Table(crtTbl.getTableName()); - tbl.setFields(crtTbl.getCols()); StorageDescriptor tblStorDesc = tbl.getTTable().getSd(); if (crtTbl.getBucketCols() != null) tblStorDesc.setBucketCols(crtTbl.getBucketCols()); @@ -169,7 +169,7 @@ List bucketCols = tbl.getBucketCols(); List sortCols = tbl.getSortCols(); - if (sortCols.size() >= bucketCols.size()) + if ( (sortCols.size() > 0) && (sortCols.size() >= bucketCols.size())) { boolean found = true; @@ -201,6 +201,10 @@ // set create time tbl.getTTable().setCreateTime((int) (System.currentTimeMillis()/1000)); + if(crtTbl.getCols() != null) { + tbl.setFields(crtTbl.getCols()); + } + // create the table db.createTable(tbl); return 0; @@ -280,6 +284,20 @@ } tbl.getTTable().getSd().setCols(alterTbl.getNewCols()); } + else if (alterTbl.getOp() == alterTableDesc.alterTableTypes.ADDPROPS) { + tbl.getTTable().getParameters().putAll(alterTbl.getProps()); + } + else if (alterTbl.getOp() == alterTableDesc.alterTableTypes.ADDSERDEPROPS) { + tbl.getTTable().getSd().getSerdeInfo().getParameters().putAll(alterTbl.getProps()); + } + else if (alterTbl.getOp() == alterTableDesc.alterTableTypes.ADDSERDE) { + tbl.setSerializationLib(alterTbl.getSerdeName()); + if ((alterTbl.getProps() != null) && (alterTbl.getProps().size() > 0)) + tbl.getTTable().getSd().getSerdeInfo().getParameters().putAll(alterTbl.getProps()); + // since serde is modified then do the appropriate things to reset columns etc + tbl.reinitSerDe(); + tbl.setFields(Hive.getFieldsFromDeserializer(tbl.getName(), tbl.getDeserializer())); + } else { console.printError("Unsupported Alter commnad"); return 1; @@ -357,7 +375,9 @@ if (col.getComment() != null) { os.write(separator); + os.write(singleQuote); os.write(col.getComment().getBytes("UTF-8")); + os.write(singleQuote); } firstCol = false; } Index: src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java =================================================================== --- src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java (revision 711472) +++ src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java (working copy) @@ -20,8 +20,11 @@ import java.io.Serializable; import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.LocalFileSystem; @@ -30,6 +33,9 @@ import org.apache.hadoop.hive.ql.plan.loadTableDesc; import org.apache.hadoop.hive.ql.plan.moveWork; import org.apache.hadoop.hive.ql.metadata.Hive; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.io.SequenceFile; +import org.apache.hadoop.mapred.SequenceFileInputFormat; import org.apache.hadoop.util.StringUtils; /** @@ -108,6 +114,44 @@ String mesg_detail = " from " + tbd.getSourceDir(); console.printInfo(mesg, mesg_detail); + // Get the file format of the table + boolean tableIsSequenceFile = tbd.getTable().getInputFileFormatClass().equals(SequenceFileInputFormat.class); + // Get all files from the src directory + FileStatus [] dirs; + ArrayList files; + try { + fs = FileSystem.get(db.getTable(tbd.getTable().getTableName()).getDataLocation(), + Hive.get().getConf()); + dirs = fs.globStatus(new Path(tbd.getSourceDir())); + files = new ArrayList(); + for (int i=0; i0) break; + } + } catch (IOException e) { + throw new HiveException("addFiles: filesystem error in check phase", e); + } + // Check if the file format of the file matches that of the table. + if (files.size() > 0) { + int fileId = 0; + boolean fileIsSequenceFile = true; + try { + SequenceFile.Reader reader = new SequenceFile.Reader( + fs, files.get(fileId).getPath(), conf); + reader.close(); + } catch (IOException e) { + fileIsSequenceFile = false; + } + if (!fileIsSequenceFile && tableIsSequenceFile) { + throw new HiveException("Cannot load text files into a table stored as SequenceFile."); + } + if (fileIsSequenceFile && !tableIsSequenceFile) { + throw new HiveException("Cannot load SequenceFiles into a table stored as TextFile."); + } + } + + if(tbd.getPartitionSpec().size() == 0) { db.loadTable(new Path(tbd.getSourceDir()), tbd.getTable().getTableName(), tbd.getReplace()); } else { Index: src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/exec/MapRedTask.java =================================================================== --- src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/exec/MapRedTask.java (revision 711472) +++ src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/exec/MapRedTask.java (working copy) @@ -62,6 +62,11 @@ String cmdLine = hadoopExec + " jar " + auxJars + " " + hiveJar + " org.apache.hadoop.hive.ql.exec.ExecDriver -plan " + planFile.toString() + " " + hiveConfArgs; + String files = ExecDriver.getRealFiles(conf); + if(!files.isEmpty()) { + cmdLine = cmdLine + " -files " + files; + } + LOG.info("Executing: " + cmdLine); Process executor = Runtime.getRuntime().exec(cmdLine); Index: src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecReducer.java =================================================================== --- src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecReducer.java (revision 711472) +++ src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecReducer.java (working copy) @@ -27,20 +27,17 @@ import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.hive.ql.plan.PlanUtils; import org.apache.hadoop.hive.ql.plan.mapredWork; import org.apache.hadoop.hive.ql.plan.tableDesc; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.exec.ExecMapper.reportStats; -import org.apache.hadoop.hive.serde2.ColumnSet; import org.apache.hadoop.hive.serde2.Deserializer; import org.apache.hadoop.hive.serde2.SerDe; import org.apache.hadoop.hive.serde2.SerDeException; -import org.apache.hadoop.hive.serde2.objectinspector.MetadataListStructObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; import org.apache.hadoop.io.BytesWritable; -import org.apache.hadoop.io.Text; +import org.apache.hadoop.io.Writable; public class ExecReducer extends MapReduceBase implements Reducer { @@ -74,15 +71,23 @@ reducer.setMapredWork(gWork); isTagged = gWork.getNeedsTagging(); try { - // We should initialize the SerDe with the TypeInfo when available. - tableDesc keyTableDesc = PlanUtils.getReduceKeyDesc(gWork); + tableDesc keyTableDesc = gWork.getKeyDesc(); inputKeyDeserializer = (SerDe)ReflectionUtils.newInstance(keyTableDesc.getDeserializerClass(), null); inputKeyDeserializer.initialize(null, keyTableDesc.getProperties()); - for(int tag=0; tag ois = new ArrayList(); + ois.add(keyObjectInspector); + ois.add(valueObjectInspector[tag]); + ois.add(ObjectInspectorFactory.getStandardPrimitiveObjectInspector(Byte.class)); + rowObjectInspector[tag] = ObjectInspectorFactory.getStandardStructObjectInspector( + Arrays.asList(fieldNames), ois); } } catch (SerDeException e) { throw new RuntimeException(e); @@ -143,18 +148,12 @@ } catch (SerDeException e) { throw new HiveException(e); } - // This is a hack for generating the correct ObjectInspector. - // In the future, we should use DynamicSerde and initialize it using the type info. - if (keyObjectInspector == null) { - // Directly create ObjectInspector here because we didn't know the number of cols till now. - keyObjectInspector = MetadataListStructObjectInspector.getInstance(((ColumnSet)keyObject).col.size()); - } // System.err.print(keyObject.toString()); while (values.hasNext()) { - Text valueText = (Text)values.next(); + Writable valueWritable = (Writable) values.next(); //System.err.print(who.getHo().toString()); try { - valueObject[tag] = inputValueDeserializer[tag].deserialize(valueText); + valueObject[tag] = inputValueDeserializer[tag].deserialize(valueWritable); } catch (SerDeException e) { throw new HiveException(e); } @@ -162,23 +161,12 @@ row.add(keyObject); row.add(valueObject[tag]); row.add(tag); - if (valueObjectInspector[tag] == null) { - // Directly create ObjectInspector here because we didn't know the number of cols till now. - valueObjectInspector[tag] = MetadataListStructObjectInspector.getInstance(((ColumnSet)valueObject[tag]).col.size()); - ArrayList ois = new ArrayList(); - ois.add(keyObjectInspector); - ois.add(valueObjectInspector[tag]); - ois.add(ObjectInspectorFactory.getStandardPrimitiveObjectInspector(Byte.class)); - rowObjectInspector[tag] = ObjectInspectorFactory.getStandardStructObjectInspector( - Arrays.asList(fieldNames), ois); - } reducer.process(row, rowObjectInspector[tag]); } - } catch (HiveException e) { abort = true; - throw new IOException (e.getMessage()); + throw new IOException (e); } } Index: src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java =================================================================== --- src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java (revision 711472) +++ src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java (working copy) @@ -219,7 +219,6 @@ return result.toArray(new HiveInputSplit[result.size()]); } - private tableDesc getTableDescFromPath(Path dir) throws IOException { partitionDesc partDesc = pathToPartitionInfo.get(dir.toString()); Index: src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/parse/ErrorMsg.java =================================================================== --- src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/parse/ErrorMsg.java (revision 711472) +++ src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/parse/ErrorMsg.java (working copy) @@ -52,14 +52,18 @@ INVALID_MAPINDEX_TYPE("Map Key Type does not Match Index Expression Type"), NON_COLLECTION_TYPE("[] not Valid on Non Collection Types"), SELECT_DISTINCT_WITH_GROUPBY("SELECT DISTINCT and GROUP BY can not be in the same query"), - COLUMN_REPAEATED_IN_PARTITIONING_COLS("Column repeated in partitioning columns"), + COLUMN_REPEATED_IN_PARTITIONING_COLS("Column repeated in partitioning columns"), DUPLICATE_COLUMN_NAMES("Duplicate column names"), COLUMN_REPEATED_IN_CLUSTER_SORT("Same column cannot appear in cluster and sort by"), SAMPLE_RESTRICTION("Cannot Sample on More Than Two Columns"), SAMPLE_COLUMN_NOT_FOUND("Sample Column Not Found"), NO_PARTITION_PREDICATE("No Partition Predicate Found"), - INVALID_DOT(". operator is only supported on struct or list of struct types"); - + INVALID_DOT(". operator is only supported on struct or list of struct types"), + INVALID_TBL_DDL_SERDE("Either list of columns or a custom serializer should be specified"), + TARGET_TABLE_COLUMN_MISMATCH("Cannot insert into target table because column number/types are different"), + TABLE_ALIAS_NOT_ALLOWED("Table Alias not Allowed in Sampling Clause"), + NON_BUCKETED_TABLE("Sampling Expression Needed for Non-Bucketed Table"); + private String mesg; ErrorMsg(String mesg) { this.mesg = mesg; Index: src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/parse/Hive.g =================================================================== --- src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/parse/Hive.g (revision 711472) +++ src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/parse/Hive.g (working copy) @@ -49,7 +49,7 @@ TOK_TRUE; TOK_FALSE; TOK_TRANSFORM; -TOK_COLLIST; +TOK_EXPLIST; TOK_ALIASLIST; TOK_GROUPBY; TOK_ORDERBY; @@ -64,6 +64,7 @@ TOK_ISNULL; TOK_ISNOTNULL; TOK_TINYINT; +TOK_SMALLINT; TOK_INT; TOK_BIGINT; TOK_BOOLEAN; @@ -81,6 +82,9 @@ TOK_ALTERTABLE_ADDCOLS; TOK_ALTERTABLE_REPLACECOLS; TOK_ALTERTABLE_DROPPARTS; +TOK_ALTERTABLE_SERDEPROPERTIES; +TOK_ALTERTABLE_SERIALIZER; +TOK_ALTERTABLE_PROPERTIES; TOK_SHOWTABLES; TOK_SHOWPARTITIONS; TOK_CREATEEXTTABLE; @@ -96,6 +100,7 @@ TOK_TABLEROWFORMATMAPKEYS; TOK_TABLEROWFORMATLINES; TOK_TBLSEQUENCEFILE; +TOK_TBLTEXTFILE; TOK_TABCOLNAME; TOK_TABLELOCATION; TOK_TABLESAMPLE; @@ -106,10 +111,10 @@ TOK_CREATEFUNCTION; TOK_EXPLAIN; TOK_TABLESERIALIZER; -TOK_TABLSERDEPROPERTIES; -TOK_TABLESERDEPROPLIST; +TOK_TABLEPROPERTIES; +TOK_TABLEPROPLIST; TOK_LIMIT; -TOKTABLESERDEPROPERTY; +TOK_TABLEPROPERTY; } @@ -157,9 +162,9 @@ ; createStatement - : KW_CREATE (ext=KW_EXTERNAL)? KW_TABLE name=Identifier LPAREN columnNameTypeList RPAREN tableComment? tablePartition? tableBuckets? tableRowFormat? tableFileFormat? tableLocation? - -> {$ext == null}? ^(TOK_CREATETABLE $name columnNameTypeList tableComment? tablePartition? tableBuckets? tableRowFormat? tableFileFormat? tableLocation?) - -> ^(TOK_CREATEEXTTABLE $name columnNameTypeList tableComment? tablePartition? tableBuckets? tableRowFormat? tableFileFormat? tableLocation?) + : KW_CREATE (ext=KW_EXTERNAL)? KW_TABLE name=Identifier (LPAREN columnNameTypeList RPAREN)? tableComment? tablePartition? tableBuckets? tableRowFormat? tableFileFormat? tableLocation? + -> {$ext == null}? ^(TOK_CREATETABLE $name columnNameTypeList? tableComment? tablePartition? tableBuckets? tableRowFormat? tableFileFormat? tableLocation?) + -> ^(TOK_CREATEEXTTABLE $name columnNameTypeList? tableComment? tablePartition? tableBuckets? tableRowFormat? tableFileFormat? tableLocation?) ; dropStatement @@ -170,6 +175,8 @@ : alterStatementRename | alterStatementAddCol | alterStatementDropPartitions + | alterStatementProperties + | alterStatementSerdeProperties ; alterStatementRename @@ -188,6 +195,18 @@ -> ^(TOK_ALTERTABLE_DROPPARTS Identifier partitionSpec+) ; +alterStatementProperties + : KW_ALTER KW_TABLE name=Identifier KW_SET KW_PROPERTIES tableProperties + -> ^(TOK_ALTERTABLE_PROPERTIES $name tableProperties) + ; + +alterStatementSerdeProperties + : KW_ALTER KW_TABLE name=Identifier KW_SET KW_SERDE serde=StringLiteral (KW_WITH KW_SERDEPROPERTIES tableProperties)? + -> ^(TOK_ALTERTABLE_SERIALIZER $name $serde tableProperties?) + | KW_ALTER KW_TABLE name=Identifier KW_SET KW_SERDEPROPERTIES tableProperties + -> ^(TOK_ALTERTABLE_SERDEPROPERTIES $name tableProperties) + ; + descStatement : KW_DESCRIBE (isExtended=KW_EXTENDED)? (tab=tabName) -> ^(TOK_DESCTABLE $tab $isExtended?) ; @@ -227,23 +246,23 @@ : KW_ROW KW_FORMAT KW_DELIMITED tableRowFormatFieldIdentifier? tableRowFormatCollItemsIdentifier? tableRowFormatMapKeysIdentifier? tableRowFormatLinesIdentifier? -> ^(TOK_TABLEROWFORMAT tableRowFormatFieldIdentifier? tableRowFormatCollItemsIdentifier? tableRowFormatMapKeysIdentifier? tableRowFormatLinesIdentifier?) - | KW_ROW KW_FORMAT KW_SERIALIZER name=StringLiteral tableSerializerProperties? - -> ^(TOK_TABLESERIALIZER $name tableSerializerProperties?) + | KW_ROW KW_FORMAT KW_SERDE name=StringLiteral (KW_WITH KW_SERDEPROPERTIES serdeprops=tableProperties)? + -> ^(TOK_TABLESERIALIZER $name $serdeprops?) ; -tableSerializerProperties +tableProperties : - KW_WITH KW_PROPERTIES LPAREN propertiesList RPAREN -> ^(TOK_TABLSERDEPROPERTIES propertiesList) + LPAREN propertiesList RPAREN -> ^(TOK_TABLEPROPERTIES propertiesList) ; propertiesList : - keyValueProperty (COMMA keyValueProperty)* -> ^(TOK_TABLESERDEPROPLIST keyValueProperty+) + keyValueProperty (COMMA keyValueProperty)* -> ^(TOK_TABLEPROPLIST keyValueProperty+) ; keyValueProperty : - key=StringLiteral EQUAL value=StringLiteral -> ^(TOKTABLESERDEPROPERTY $key $value) + key=StringLiteral EQUAL value=StringLiteral -> ^(TOK_TABLEPROPERTY $key $value) ; tableRowFormatFieldIdentifier @@ -273,6 +292,7 @@ tableFileFormat : KW_STORED KW_AS KW_SEQUENCEFILE -> TOK_TBLSEQUENCEFILE + | KW_STORED KW_AS KW_TEXTFILE -> TOK_TBLTEXTFILE ; tableLocation @@ -317,6 +337,7 @@ primitiveType : KW_TINYINT -> TOK_TINYINT + | KW_SMALLINT -> TOK_SMALLINT | KW_INT -> TOK_INT | KW_BIGINT -> TOK_BIGINT | KW_BOOLEAN -> TOK_BOOLEAN @@ -420,23 +441,22 @@ selectList : - selectItem - ( COMMA selectItem )* -> selectItem+ + selectItem ( COMMA selectItem )* -> selectItem+ + | trfmClause -> ^(TOK_SELEXPR trfmClause) ; selectItem : - trfmClause -> ^(TOK_SELEXPR trfmClause) - | (selectExpression (KW_AS Identifier)?) -> ^(TOK_SELEXPR selectExpression Identifier?) + ( selectExpression (KW_AS Identifier)?) -> ^(TOK_SELEXPR selectExpression Identifier?) ; trfmClause : KW_TRANSFORM - LPAREN columnList RPAREN + LPAREN expressionList RPAREN KW_AS LPAREN aliasList RPAREN - KW_USING StringLiteral -> ^(TOK_TRANSFORM columnList aliasList StringLiteral) + KW_USING StringLiteral -> ^(TOK_TRANSFORM expressionList aliasList StringLiteral) ; selectExpression @@ -454,12 +474,12 @@ // table.column tableColumn : - (tab=Identifier)? DOT col=Identifier -> ^(TOK_COLREF $tab? $col) + (tab=Identifier DOT)? col=Identifier -> ^(TOK_COLREF $tab? $col) ; -columnList +expressionList : - tableColumn (COMMA tableColumn)* -> ^(TOK_COLLIST tableColumn+) + expression (COMMA expression)* -> ^(TOK_EXPLIST expression+) ; aliasList @@ -496,7 +516,7 @@ tableSample : - KW_TABLESAMPLE LPAREN KW_BUCKET (numerator=Number) KW_OUT KW_OF (denominator=Number) (KW_ON col+=Identifier (COMMA col+=Identifier)*)? RPAREN -> ^(TOK_TABLESAMPLE $numerator $denominator $col*) + KW_TABLESAMPLE LPAREN KW_BUCKET (numerator=Number) KW_OUT KW_OF (denominator=Number) (KW_ON expr+=expression (COMMA expr+=expression)*)? RPAREN -> ^(TOK_TABLESAMPLE $numerator $denominator $expr*) ; tableSource @@ -570,12 +590,12 @@ : // LEFT and RIGHT keywords are also function names Identifier LPAREN ( - (dist=KW_DISTINCT)? - expression - (COMMA expression)* + ((dist=KW_DISTINCT)? + expression + (COMMA expression)*)? )? - RPAREN -> {$dist == null}? ^(TOK_FUNCTION Identifier expression+) - -> ^(TOK_FUNCTIONDI Identifier expression+) + RPAREN -> {$dist == null}? ^(TOK_FUNCTION Identifier (expression+)?) + -> ^(TOK_FUNCTIONDI Identifier (expression+)?) ; @@ -644,7 +664,7 @@ precedenceUnaryExpression (precedenceBitwiseXorOperator^ precedenceUnaryExpression)* ; - + precedenceStarOperator : STAR | DIVIDE | MOD @@ -808,6 +828,7 @@ KW_COMMENT: 'COMMENT'; KW_BOOLEAN: 'BOOLEAN'; KW_TINYINT: 'TINYINT'; +KW_SMALLINT: 'SMALLINT'; KW_INT: 'INT'; KW_BIGINT: 'BIGINT'; KW_FLOAT: 'FLOAT'; @@ -834,6 +855,7 @@ KW_LINES: 'LINES'; KW_STORED: 'STORED'; KW_SEQUENCEFILE: 'SEQUENCEFILE'; +KW_TEXTFILE: 'TEXTFILE'; KW_LOCATION: 'LOCATION'; KW_TABLESAMPLE: 'TABLESAMPLE'; KW_BUCKET: 'BUCKET'; @@ -849,10 +871,12 @@ KW_FUNCTION: 'FUNCTION'; KW_EXPLAIN: 'EXPLAIN'; KW_EXTENDED: 'EXTENDED'; -KW_SERIALIZER: 'SERIALIZER'; +KW_SERDE: 'SERDE'; KW_WITH: 'WITH'; -KW_PROPERTIES: 'SERDEPROPERTIES'; +KW_SERDEPROPERTIES: 'SERDEPROPERTIES'; KW_LIMIT: 'LIMIT'; +KW_SET: 'SET'; +KW_PROPERTIES: 'TBLPROPERTIES'; // Operators @@ -909,7 +933,7 @@ StringLiteral : - '\'' (~'\'')* '\'' ( '\'' (~'\'')* '\'' )* + ( '\'' (~'\'')* '\'' | '\"' (~'\"')* '\"' )+ ; CharSetLiteral @@ -926,6 +950,7 @@ Identifier : (Letter | Digit) (Letter | Digit | '_')* + | '`' (Letter | Digit) (Letter | Digit | '_')* '`' ; CharSetName Index: src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzerFactory.java =================================================================== --- src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzerFactory.java (revision 711472) +++ src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzerFactory.java (working copy) @@ -39,6 +39,9 @@ case HiveParser.TOK_ALTERTABLE_REPLACECOLS: case HiveParser.TOK_ALTERTABLE_RENAME: case HiveParser.TOK_ALTERTABLE_DROPPARTS: + case HiveParser.TOK_ALTERTABLE_PROPERTIES: + case HiveParser.TOK_ALTERTABLE_SERIALIZER: + case HiveParser.TOK_ALTERTABLE_SERDEPROPERTIES: case HiveParser.TOK_SHOWTABLES: case HiveParser.TOK_SHOWPARTITIONS: return new DDLSemanticAnalyzer(conf); Index: src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/parse/QB.java =================================================================== --- src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/parse/QB.java (revision 711472) +++ src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/parse/QB.java (working copy) @@ -22,7 +22,6 @@ import org.apache.hadoop.hive.ql.parse.QBParseInfo; import org.apache.hadoop.hive.ql.parse.QBMetaData; -import org.apache.hadoop.hive.ql.metadata.Table; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -158,14 +157,6 @@ } public boolean isSelectStarQuery() { - if (!qbp.isSelectStarQuery() || !aliasToSubq.isEmpty()) - return false; - - Iterator> iter = qbm.getAliasToTable().entrySet().iterator(); - Table tab = ((Map.Entry)iter.next()).getValue(); - if (tab.isPartitioned()) - return false; - - return true; + return qbp.isSelectStarQuery() && aliasToSubq.isEmpty(); } } Index: src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/parse/QBMetaData.java =================================================================== --- src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/parse/QBMetaData.java (revision 711472) +++ src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/parse/QBMetaData.java (working copy) @@ -115,5 +115,4 @@ public Table getSrcForAlias(String alias) { return this.aliasToTable.get(alias.toLowerCase()); } - } Index: src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java =================================================================== --- src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java (revision 711472) +++ src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java (working copy) @@ -104,9 +104,10 @@ } public static String stripQuotes(String val) throws SemanticException { - if (val.charAt(0) == '\'' && val.charAt(val.length() - 1) == '\'') { + if ((val.charAt(0) == '\'' && val.charAt(val.length() - 1) == '\'') + || (val.charAt(0) == '\"' && val.charAt(val.length() - 1) == '\"')) { val = val.substring(1, val.length() - 1); - } + } return val; } @@ -142,19 +143,48 @@ } } + /** + * Remove the encapsulating "`" pair from the identifier. + * We allow users to use "`" to escape identifier for table names, + * column names and aliases, in case that coincide with Hive language + * keywords. + */ + public static String unescapeIdentifier(String val) { + if (val == null) { + return null; + } + if (val.charAt(0) == '`' && val.charAt(val.length() - 1) == '`') { + val = val.substring(1, val.length() - 1); + } + return val; + } + @SuppressWarnings("nls") public static String unescapeSQLString(String b) { - assert(b.charAt(0) == '\''); - assert(b.charAt(b.length()-1) == '\''); + Character enclosure = null; + // Some of the strings can be passed in as unicode. For example, the // delimiter can be passed in as \002 - So, we first check if the // string is a unicode number, else go back to the old behavior StringBuilder sb = new StringBuilder(b.length()); - int i = 1; - while (i < (b.length()-1)) { - - if (b.charAt(i) == '\\' && (i+4 < b.length())) { + for (int i=0; i < b.length(); i++) { + + char currentChar = b.charAt(i); + if (enclosure == null) { + if (currentChar == '\'' || b.charAt(i) == '\"') { + enclosure = currentChar; + } + // ignore all other chars outside the enclosure + continue; + } + + if (enclosure.equals(currentChar)) { + enclosure = null; + continue; + } + + if (currentChar == '\\' && (i+4 < b.length())) { char i1 = b.charAt(i+1); char i2 = b.charAt(i+2); char i3 = b.charAt(i+3); @@ -167,12 +197,12 @@ bValArr[0] = bVal; String tmp = new String(bValArr); sb.append(tmp); - i += 4; + i += 3; continue; } } - - if (b.charAt(i) == '\\' && (i+2 < b.length())) { + + if (currentChar == '\\' && (i+2 < b.length())) { char n=b.charAt(i+1); switch(n) { case '0': sb.append("\0"); break; @@ -191,9 +221,8 @@ } i++; } else { - sb.append(b.charAt(i)); + sb.append(currentChar); } - i++; } return sb.toString(); } @@ -219,7 +248,7 @@ try { // get table metadata - tableName = ast.getChild(0).getText(); + tableName = unescapeIdentifier(ast.getChild(0).getText()); tableHandle = db.getTable(tableName); // get partition metadata if partition specified @@ -230,7 +259,7 @@ for (int i = 0; i < partspec.getChildCount(); ++i) { CommonTree partspec_val = (CommonTree) partspec.getChild(i); String val = stripQuotes(partspec_val.getChild(1).getText()); - partSpec.put(partspec_val.getChild(0).getText(), val); + partSpec.put(unescapeIdentifier(partspec_val.getChild(0).getText()), val); } partHandle = Hive.get().getPartition(tableHandle, partSpec, forceCreatePartition); if(partHandle == null) { Index: src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/parse/PartitionPruner.java =================================================================== --- src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/parse/PartitionPruner.java (revision 711472) +++ src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/parse/PartitionPruner.java (working copy) @@ -59,6 +59,12 @@ private Table tab; private exprNodeDesc prunerExpr; + + // is set to true if the expression only contains partitioning columns and not any other column reference. + // This is used to optimize select * from table where ... scenario, when the where condition only references + // partitioning columns - the partitions are identified and streamed directly to the client without requiring + // a map-reduce job + private boolean containsPartCols; /** Creates a new instance of PartitionPruner */ public PartitionPruner(String tableAlias, QBMetaData metaData) { @@ -66,8 +72,13 @@ this.metaData = metaData; this.tab = metaData.getTableForAlias(tableAlias); this.prunerExpr = null; + containsPartCols = true; } + public boolean containsPartitionCols() { + return containsPartCols; + } + /** * We use exprNodeConstantDesc(class,null) to represent unknown values. * Except UDFOPAnd, UDFOPOr, and UDFOPNot, all UDFs are assumed to return unknown values @@ -97,12 +108,18 @@ switch (tokType) { case HiveParser.TOK_COLREF: { - assert(expr.getChildCount() == 2); - String tabAlias = expr.getChild(0).getText(); - String colName = expr.getChild(1).getText(); - if (tabAlias == null || colName == null) { - throw new SemanticException(ErrorMsg.INVALID_XPATH.getMsg(expr)); + String tabAlias = null; + String colName = null; + if (expr.getChildCount() != 1) { + assert(expr.getChildCount() == 2); + tabAlias = BaseSemanticAnalyzer.unescapeIdentifier(expr.getChild(0).getText()); + colName = BaseSemanticAnalyzer.unescapeIdentifier(expr.getChild(1).getText()); } + else { + colName = BaseSemanticAnalyzer.unescapeIdentifier(expr.getChild(0).getText()); + tabAlias = SemanticAnalyzer.getTabAliasForCol(this.metaData, colName, (CommonTree)expr.getChild(0)); + } + // Set value to null if it's not partition column if (tabAlias.equals(tableAlias) && tab.isPartitionKey(colName)) { desc = new exprNodeColumnDesc(String.class, colName); @@ -117,6 +134,7 @@ TypeInfo typeInfo = TypeInfoUtils.getTypeInfoFromObjectInspector( this.metaData.getTableForAlias(tabAlias).getDeserializer().getObjectInspector()); desc = new exprNodeConstantDesc(typeInfo.getStructFieldTypeInfo(colName), null); + containsPartCols = false; } } catch (SerDeException e){ throw new RuntimeException(e); @@ -195,8 +213,8 @@ case HiveParser.TOK_COLREF: { assert(expr.getChildCount() == 2); - String tabAlias = expr.getChild(0).getText(); - String colName = expr.getChild(1).getText(); + String tabAlias = BaseSemanticAnalyzer.unescapeIdentifier(expr.getChild(0).getText()); + String colName = BaseSemanticAnalyzer.unescapeIdentifier(expr.getChild(1).getText()); if (tabAlias.equals(tableAlias) && tab.isPartitionKey(colName)) { hasPPred = true; } Index: src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java =================================================================== --- src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java (revision 711472) +++ src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java (working copy) @@ -24,6 +24,7 @@ import java.lang.reflect.Method; import org.antlr.runtime.tree.*; +import org.apache.hadoop.hive.serde2.Deserializer; import org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe; import org.apache.hadoop.hive.serde2.SerDeException; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; @@ -33,6 +34,7 @@ import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.ql.Context; +import org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat; import org.apache.hadoop.hive.ql.metadata.*; import org.apache.hadoop.hive.ql.plan.*; import org.apache.hadoop.hive.ql.typeinfo.TypeInfo; @@ -41,6 +43,7 @@ import org.apache.hadoop.hive.ql.udf.UDFOPPositive; import org.apache.hadoop.hive.ql.exec.*; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hadoop.mapred.TextInputFormat; import org.apache.hadoop.fs.Path; @@ -142,7 +145,7 @@ || expressionTree.getToken().getType() == HiveParser.TOK_FUNCTIONDI) { assert (expressionTree.getChildCount() != 0); assert (expressionTree.getChild(0).getType() == HiveParser.Identifier); - String functionName = expressionTree.getChild(0).getText(); + String functionName = unescapeIdentifier(expressionTree.getChild(0).getText()); if (FunctionRegistry.getUDAF(functionName) != null) { aggregations.put(expressionTree.toStringTree(), expressionTree); return; @@ -195,17 +198,17 @@ tableSamplePresent = true; } CommonTree tableTree = (CommonTree)(tabref.getChild(0)); - String alias = tabref.getChild(aliasIndex).getText(); + String alias = unescapeIdentifier(tabref.getChild(aliasIndex).getText()); // If the alias is already there then we have a conflict if (qb.exists(alias)) { throw new SemanticException(ErrorMsg.AMBIGOUS_TABLE_ALIAS.getMsg(tabref.getChild(aliasIndex))); } if (tableSamplePresent) { CommonTree sampleClause = (CommonTree)tabref.getChild(1); - ArrayList sampleCols = new ArrayList(); + ArrayList sampleCols = new ArrayList(); if (sampleClause.getChildCount() > 2) { for (int i = 2; i < sampleClause.getChildCount(); i++) { - sampleCols.add(sampleClause.getChild(i).getText()); + sampleCols.add((CommonTree)sampleClause.getChild(i)); } } // TODO: For now only support sampling on up to two columns @@ -214,13 +217,13 @@ throw new SemanticException(ErrorMsg.SAMPLE_RESTRICTION.getMsg(tabref.getChild(0))); } qb.getParseInfo().setTabSample(alias, new TableSample( - sampleClause.getChild(0).getText(), - sampleClause.getChild(1).getText(), - sampleCols) + unescapeIdentifier(sampleClause.getChild(0).getText()), + unescapeIdentifier(sampleClause.getChild(1).getText()), + sampleCols) ); } // Insert this map into the stats - String table_name = tabref.getChild(0).getText(); + String table_name = unescapeIdentifier(tabref.getChild(0).getText()); qb.setTabAlias(alias, table_name); qb.getParseInfo().setSrcForAlias(alias, tableTree); @@ -233,7 +236,7 @@ throw new SemanticException(ErrorMsg.NO_SUBQUERY_ALIAS.getMsg(subq)); } CommonTree subqref = (CommonTree) subq.getChild(0); - String alias = subq.getChild(1).getText(); + String alias = unescapeIdentifier(subq.getChild(1).getText()); // Recursively do the first phase of semantic analysis for the subquery QBExpr qbexpr = new QBExpr(alias); @@ -572,7 +575,7 @@ // String[] allAliases = joinTree.getAllAliases(); switch (condn.getToken().getType()) { case HiveParser.TOK_COLREF: - String tblName = condn.getChild(0).getText(); + String tblName = unescapeIdentifier(condn.getChild(0).getText()); if (isPresent(joinTree.getLeftAliases(), tblName)) { if (!leftAliases.contains(tblName)) leftAliases.add(tblName); @@ -673,8 +676,7 @@ OperatorInfo output = (OperatorInfo)input.clone(); output.setOp( OperatorFactory.getAndMakeChild( - new filterDesc(genExprNodeDesc((CommonTree)whereExpr.getChild(0), - qb.getParseInfo().getAlias(), + new filterDesc(genExprNodeDesc(qb.getMetaData(), (CommonTree)whereExpr.getChild(0), input.getRowResolver())), new RowSchema(output.getRowResolver().getColumnInfos()), input.getOp() @@ -703,6 +705,37 @@ } } + /** + * If the user script command needs any modifications - do it here + */ + private String getFixedCmd(String cmd) { + SessionState ss = SessionState.get(); + if(ss == null) + return cmd; + + // for local mode - replace any references to packaged files by name with + // the reference to the original file path + if(ss.getConf().get("mapred.job.tracker", "local").equals("local")) { + Set files = ss.list_resource(SessionState.ResourceType.FILE, null); + if((files != null) && !files.isEmpty()) { + int end = cmd.indexOf(" "); + String prog = (end == -1) ? cmd : cmd.substring(0, end); + String args = (end == -1) ? "" : cmd.substring(end, cmd.length()); + + for(String oneFile: files) { + Path p = new Path(oneFile); + if(p.getName().equals(prog)) { + cmd = oneFile + args; + break; + } + } + } + } + + return cmd; + } + + @SuppressWarnings("nls") private OperatorInfo genScriptPlan(CommonTree trfm, QB qb, OperatorInfo input) throws SemanticException { @@ -719,11 +752,11 @@ if (i != 0) { sb.append(","); } - sb.append(((CommonTree)collist.getChild(i)).getText()); + sb.append(unescapeIdentifier(((CommonTree)collist.getChild(i)).getText())); out_rwsch.put( qb.getParseInfo().getAlias(), - ((CommonTree)collist.getChild(i)).getText(), - new ColumnInfo(((CommonTree)collist.getChild(i)).getText(), + unescapeIdentifier(((CommonTree)collist.getChild(i)).getText()), + new ColumnInfo(unescapeIdentifier(((CommonTree)collist.getChild(i)).getText()), String.class) // Everything is a string right now ); } @@ -732,7 +765,7 @@ .setOp(OperatorFactory .getAndMakeChild( new scriptDesc( - stripQuotes(trfm.getChild(2).getText()), + getFixedCmd(stripQuotes(trfm.getChild(2).getText())), PlanUtils.getDefaultTableDesc(Integer.toString(Utilities.tabCode), sb.toString()), PlanUtils.getDefaultTableDesc(Integer.toString(Utilities.tabCode), "")), new RowSchema( @@ -776,17 +809,22 @@ private static String getColAlias(CommonTree selExpr, String defaultName) { if (selExpr.getChildCount() == 2) { // return zz for "xx + yy AS zz" - return selExpr.getChild(1).getText(); + return unescapeIdentifier(selExpr.getChild(1).getText()); } CommonTree root = (CommonTree)selExpr.getChild(0); while (root.getType() == HiveParser.DOT || root.getType() == HiveParser.TOK_COLREF) { - assert(root.getChildCount() == 2); - root = (CommonTree) root.getChild(1); + if (root.getType() == HiveParser.TOK_COLREF && root.getChildCount() == 1) { + root = (CommonTree) root.getChild(0); + } + else { + assert(root.getChildCount() == 2); + root = (CommonTree) root.getChild(1); + } } if (root.getType() == HiveParser.Identifier) { // Return zz for "xx.zz" and "xx.yy.zz" - return root.getText(); + return unescapeIdentifier(root.getText()); } else { // Return defaultName if selExpr is not a simple xx.yy.zz return defaultName; @@ -829,22 +867,21 @@ col_list, input.getRowResolver(), pos, out_rwsch); } else { - exprNodeDesc exp = genExprNodeDesc(expr, alias, input.getRowResolver()); + exprNodeDesc exp = genExprNodeDesc(qb.getMetaData(), expr, input.getRowResolver()); col_list.add(exp); if (!StringUtils.isEmpty(alias) && (out_rwsch.get(alias, colAlias) != null)) { throw new SemanticException(ErrorMsg.AMBIGOUS_COLUMN.getMsg(expr.getChild(1))); } - out_rwsch.put(alias, expr.getText(), + out_rwsch.put(alias, unescapeIdentifier(expr.getText()), new ColumnInfo((Integer.valueOf(pos)).toString(), - exp.getTypeInfo())); // Everything is a string right now + exp.getTypeInfo())); } } } else { // Case when this is an expression - exprNodeDesc exp = genExprNodeDesc(sel, qb.getParseInfo() - .getAlias(), input.getRowResolver()); + exprNodeDesc exp = genExprNodeDesc(qb.getMetaData(), sel, input.getRowResolver()); col_list.add(exp); if (!StringUtils.isEmpty(alias) && (out_rwsch.get(alias, colAlias) != null)) { @@ -854,7 +891,7 @@ // of the expression as the column name out_rwsch.put(alias, colAlias, new ColumnInfo((Integer.valueOf(pos)).toString(), - exp.getTypeInfo())); // Everything is a string right now + exp.getTypeInfo())); } pos = Integer.valueOf(pos.intValue() + 1); } @@ -882,6 +919,45 @@ return output; } + /** + * Class to store UDAF related information. + */ + static class UDAFInfo { + ArrayList convertedParameters; + Method aggregateMethod; + Method evaluateMethod; + } + + /** + * Returns the UDAFInfo struct for the aggregation + * @param aggName The name of the UDAF. + * @param mode The mode of the aggregation. This affects the evaluate method. + * @param aggClasses The classes of the parameters to the UDAF. + * @param aggParameters The actual exprNodeDesc of the parameters. + * @param aggTree The CommonTree node of the UDAF in the query. + * @return UDAFInfo + * @throws SemanticException when the UDAF is not found or has problems. + */ + UDAFInfo getUDAFInfo(String aggName, groupByDesc.Mode mode, ArrayList> aggClasses, + ArrayList aggParameters, CommonTree aggTree) throws SemanticException { + UDAFInfo r = new UDAFInfo(); + r.aggregateMethod = FunctionRegistry.getUDAFMethod(aggName, aggClasses); + if (null == r.aggregateMethod) { + String reason = "Looking for UDAF \"" + aggName + "\" with parameters " + aggClasses; + throw new SemanticException(ErrorMsg.INVALID_FUNCTION_SIGNATURE.getMsg((CommonTree)aggTree.getChild(0), reason)); + } + + r.convertedParameters = convertParameters(r.aggregateMethod, aggParameters); + + r.evaluateMethod = FunctionRegistry.getUDAFEvaluateMethod(aggName, mode); + if (r.evaluateMethod == null) { + String reason = "UDAF \"" + aggName + "\" does not have evaluate()/evaluatePartial() methods."; + throw new SemanticException(ErrorMsg.INVALID_FUNCTION.getMsg((CommonTree)aggTree.getChild(0), reason)); + } + + return r; + } + @SuppressWarnings("nls") private OperatorInfo genGroupByPlanGroupByOperator( QBParseInfo parseInfo, String dest, OperatorInfo reduceSinkOperatorInfo, @@ -933,16 +1009,13 @@ aggClasses.add(paraExprInfo.getType().getPrimitiveClass()); } - if (null == FunctionRegistry.getUDAFMethod(aggName, aggClasses)) { - String reason = "Looking for UDAF \"" + aggName + "\" with parameters " + aggClasses; - throw new SemanticException(ErrorMsg.INVALID_FUNCTION_SIGNATURE.getMsg((CommonTree)value.getChild(0), reason)); - } + UDAFInfo udaf = getUDAFInfo(aggName, mode, aggClasses, aggParameters, value); - aggregations.add(new aggregationDesc(aggClass, aggParameters, + aggregations.add(new aggregationDesc(aggClass, udaf.convertedParameters, value.getToken().getType() == HiveParser.TOK_FUNCTIONDI)); groupByOutputRowResolver.put("",value.toStringTree(), new ColumnInfo(Integer.valueOf(groupByKeys.size() + aggregations.size() -1).toString(), - String.class)); // Everything is a string right now + udaf.evaluateMethod.getReturnType())); } return new OperatorInfo( @@ -954,6 +1027,209 @@ } @SuppressWarnings("nls") + private OperatorInfo genGroupByPlanGroupByOperator1( + QBParseInfo parseInfo, String dest, OperatorInfo reduceSinkOperatorInfo, + groupByDesc.Mode mode) + throws SemanticException { + RowResolver groupByInputRowResolver = reduceSinkOperatorInfo.getRowResolver(); + RowResolver groupByOutputRowResolver = new RowResolver(); + groupByOutputRowResolver.setIsExprResolver(true); + ArrayList groupByKeys = new ArrayList(); + ArrayList aggregations = new ArrayList(); + List grpByExprs = getGroupByForClause(parseInfo, dest); + for (int i = 0; i < grpByExprs.size(); ++i) { + CommonTree grpbyExpr = grpByExprs.get(i); + String text = grpbyExpr.toStringTree(); + ColumnInfo exprInfo = groupByInputRowResolver.get("",text); + + if (exprInfo == null) { + throw new SemanticException(ErrorMsg.INVALID_COLUMN.getMsg(grpbyExpr)); + } + + groupByKeys.add(new exprNodeColumnDesc(exprInfo.getType(), exprInfo.getInternalName())); + String field = (Integer.valueOf(i)).toString(); + groupByOutputRowResolver.put("",grpbyExpr.toStringTree(), + new ColumnInfo(field, exprInfo.getType())); + } + + // If there is a distinctFuncExp, add all parameters to the reduceKeys. + if (parseInfo.getDistinctFuncExprForClause(dest) != null) { + CommonTree value = parseInfo.getDistinctFuncExprForClause(dest); + String aggName = value.getChild(0).getText(); + Class aggClass = FunctionRegistry.getUDAF(aggName); + assert (aggClass != null); + ArrayList aggParameters = new ArrayList(); + ArrayList> aggClasses = new ArrayList>(); + // 0 is the function name + for (int i = 1; i < value.getChildCount(); i++) { + String text = value.getChild(i).toStringTree(); + CommonTree paraExpr = (CommonTree)value.getChild(i); + ColumnInfo paraExprInfo = groupByInputRowResolver.get("",text); + if (paraExprInfo == null) { + throw new SemanticException(ErrorMsg.INVALID_COLUMN.getMsg(paraExpr)); + } + + String paraExpression = paraExprInfo.getInternalName(); + assert(paraExpression != null); + aggParameters.add(new exprNodeColumnDesc(paraExprInfo.getType(), paraExprInfo.getInternalName())); + aggClasses.add(paraExprInfo.getType().getPrimitiveClass()); + } + + UDAFInfo udaf = getUDAFInfo(aggName, mode, aggClasses, aggParameters, value); + + aggregations.add(new aggregationDesc(aggClass, udaf.convertedParameters, true)); + groupByOutputRowResolver.put("",value.toStringTree(), + new ColumnInfo(Integer.valueOf(groupByKeys.size() + aggregations.size() -1).toString(), + udaf.evaluateMethod.getReturnType())); + } + + HashMap aggregationTrees = parseInfo + .getAggregationExprsForClause(dest); + for (Map.Entry entry : aggregationTrees.entrySet()) { + CommonTree value = entry.getValue(); + if (value.getToken().getType() == HiveParser.TOK_FUNCTIONDI) + continue; + + String aggName = value.getChild(0).getText(); + Class aggClass = FunctionRegistry.getUDAF(aggName); + assert (aggClass != null); + ArrayList aggParameters = new ArrayList(); + String text = entry.getKey(); + ColumnInfo paraExprInfo = groupByInputRowResolver.get("",text); + if (paraExprInfo == null) { + throw new SemanticException(ErrorMsg.INVALID_COLUMN.getMsg(value)); + } + String paraExpression = paraExprInfo.getInternalName(); + assert(paraExpression != null); + aggParameters.add(new exprNodeColumnDesc(paraExprInfo.getType(), paraExpression)); + aggregations.add(new aggregationDesc(aggClass, aggParameters, ((mode == groupByDesc.Mode.FINAL) ? false : (value.getToken().getType() == HiveParser.TOK_FUNCTIONDI)))); + groupByOutputRowResolver.put("", value.toStringTree(), + new ColumnInfo(Integer.valueOf(groupByKeys.size() + aggregations.size() - 1).toString(), + paraExprInfo.getType())); + } + + return new OperatorInfo( + OperatorFactory.getAndMakeChild(new groupByDesc(mode, groupByKeys, aggregations), + new RowSchema(groupByOutputRowResolver.getColumnInfos()), + reduceSinkOperatorInfo.getOp()), + groupByOutputRowResolver + ); + } + + @SuppressWarnings("nls") + private OperatorInfo genGroupByPlanMapGroupByOperator( + QB qb, String dest, OperatorInfo inputOperatorInfo, + groupByDesc.Mode mode) + throws SemanticException { + QBParseInfo parseInfo = qb.getParseInfo(); + RowResolver groupByInputRowResolver = inputOperatorInfo.getRowResolver(); + RowResolver groupByOutputRowResolver = new RowResolver(); + groupByOutputRowResolver.setIsExprResolver(true); + ArrayList groupByKeys = new ArrayList(); + ArrayList aggregations = new ArrayList(); + List grpByExprs = getGroupByForClause(parseInfo, dest); + for (int i = 0; i < grpByExprs.size(); ++i) { + CommonTree grpbyExpr = grpByExprs.get(i); + exprNodeDesc grpByExprNode = genExprNodeDesc(qb.getMetaData(), grpbyExpr, groupByInputRowResolver); + + groupByKeys.add(grpByExprNode); + String field = (Integer.valueOf(i)).toString(); + groupByOutputRowResolver.put("",grpbyExpr.toStringTree(), + new ColumnInfo(field, grpByExprNode.getTypeInfo())); + } + + // If there is a distinctFuncExp, add all parameters to the reduceKeys. + if (parseInfo.getDistinctFuncExprForClause(dest) != null) { + CommonTree value = parseInfo.getDistinctFuncExprForClause(dest); + int numDistn=0; + // 0 is function name + for (int i = 1; i < value.getChildCount(); i++) { + CommonTree parameter = (CommonTree) value.getChild(i); + String text = parameter.toStringTree(); + if (groupByOutputRowResolver.get("",text) == null) { + exprNodeDesc distExprNode = genExprNodeDesc(qb.getMetaData(), parameter, groupByInputRowResolver); + groupByKeys.add(distExprNode); + numDistn++; + String field = (Integer.valueOf(grpByExprs.size() + numDistn -1)).toString(); + groupByOutputRowResolver.put("", text, new ColumnInfo(field, distExprNode.getTypeInfo())); + } + } + } + + // For each aggregation + HashMap aggregationTrees = parseInfo + .getAggregationExprsForClause(dest); + assert (aggregationTrees != null); + + for (Map.Entry entry : aggregationTrees.entrySet()) { + CommonTree value = entry.getValue(); + String aggName = value.getChild(0).getText(); + Class aggClass = FunctionRegistry.getUDAF(aggName); + assert (aggClass != null); + ArrayList aggParameters = new ArrayList(); + ArrayList> aggClasses = new ArrayList>(); + // 0 is the function name + for (int i = 1; i < value.getChildCount(); i++) { + CommonTree paraExpr = (CommonTree)value.getChild(i); + exprNodeDesc paraExprNode = genExprNodeDesc(qb.getMetaData(), paraExpr, groupByInputRowResolver); + + aggParameters.add(paraExprNode); + aggClasses.add(paraExprNode.getTypeInfo().getPrimitiveClass()); + } + + UDAFInfo udaf = getUDAFInfo(aggName, mode, aggClasses, aggParameters, value); + + aggregations.add(new aggregationDesc(aggClass, udaf.convertedParameters, + value.getToken().getType() == HiveParser.TOK_FUNCTIONDI)); + groupByOutputRowResolver.put("",value.toStringTree(), + new ColumnInfo(Integer.valueOf(groupByKeys.size() + aggregations.size() -1).toString(), + udaf.evaluateMethod.getReturnType())); + } + + return new OperatorInfo( + OperatorFactory.getAndMakeChild(new groupByDesc(mode, groupByKeys, aggregations), + new RowSchema(groupByOutputRowResolver.getColumnInfos()), + inputOperatorInfo.getOp()), + groupByOutputRowResolver); + } + + private ArrayList convertParameters(Method m, ArrayList aggParameters) { + + ArrayList newParameters = new ArrayList(); + Class[] pTypes = m.getParameterTypes(); + + // 0 is the function name + for (int i = 0; i < aggParameters.size(); i++) { + exprNodeDesc desc = aggParameters.get(i); + Class pType = ObjectInspectorUtils.generalizePrimitive(pTypes[i]); + if (desc instanceof exprNodeNullDesc) { + exprNodeConstantDesc newCh = new exprNodeConstantDesc(TypeInfoFactory.getPrimitiveTypeInfo(pType), null); + newParameters.add(newCh); + } else if (pType.isAssignableFrom(desc.getTypeInfo().getPrimitiveClass())) { + // no type conversion needed + newParameters.add(desc); + } else { + // must be implicit type conversion + Class from = desc.getTypeInfo().getPrimitiveClass(); + Class to = pType; + assert(FunctionRegistry.implicitConvertable(from, to)); + Method conv = FunctionRegistry.getUDFMethod(to.getName(), true, from); + assert(conv != null); + Class c = FunctionRegistry.getUDFClass(to.getName()); + assert(c != null); + + // get the conversion method + ArrayList conversionArg = new ArrayList(1); + conversionArg.add(desc); + newParameters.add(new exprNodeFuncDesc(TypeInfoFactory.getPrimitiveTypeInfo(pType), + c, conv, conversionArg)); + } + } + + return newParameters; + } + + @SuppressWarnings("nls") private OperatorInfo genGroupByPlanGroupByOpForward( QBParseInfo parseInfo, String dest, OperatorInfo forwardOpInfo, groupByDesc.Mode mode) @@ -1005,16 +1281,13 @@ aggClasses.add(paraExprInfo.getType().getPrimitiveClass()); } - if (null == FunctionRegistry.getUDAFMethod(aggName, aggClasses)) { - String reason = "Looking for UDAF \"" + aggName + "\" with parameters " + aggClasses; - throw new SemanticException(ErrorMsg.INVALID_FUNCTION_SIGNATURE.getMsg((CommonTree)value.getChild(0), reason)); - } + UDAFInfo udaf = getUDAFInfo(aggName, mode, aggClasses, aggParameters, value); - aggregations.add(new aggregationDesc(aggClass, aggParameters, + aggregations.add(new aggregationDesc(aggClass, udaf.convertedParameters, value.getToken().getType() == HiveParser.TOK_FUNCTIONDI)); outputRS.put("",value.toStringTree(), new ColumnInfo(Integer.valueOf(groupByKeys.size() + aggregations.size() -1).toString(), - String.class)); // Everything is a string right now + udaf.evaluateMethod.getReturnType())); } return new OperatorInfo( @@ -1027,8 +1300,76 @@ @SuppressWarnings("nls") private OperatorInfo genGroupByPlanReduceSinkOperator(QBParseInfo parseInfo, + String dest, OperatorInfo inputOperatorInfo) + throws SemanticException { + RowResolver reduceSinkInputRowResolver = inputOperatorInfo.getRowResolver(); + RowResolver reduceSinkOutputRowResolver = new RowResolver(); + reduceSinkOutputRowResolver.setIsExprResolver(true); + ArrayList reduceKeys = new ArrayList(); + + // Pre-compute group-by keys and store in reduceKeys + List grpByExprs = getGroupByForClause(parseInfo, dest); + for (int i = 0; i < grpByExprs.size(); ++i) { + CommonTree grpbyExpr = grpByExprs.get(i); + String text = grpbyExpr.toStringTree(); + + if (reduceSinkOutputRowResolver.get("", text) == null) { + ColumnInfo exprInfo = reduceSinkInputRowResolver.get("", text); + reduceKeys.add(new exprNodeColumnDesc(exprInfo.getType(), exprInfo.getInternalName())); + reduceSinkOutputRowResolver.put("", text, + new ColumnInfo(Utilities.ReduceField.KEY.toString() + "." + Integer.valueOf(reduceKeys.size() - 1).toString(), + exprInfo.getType())); + } + } + + // If there is a distinctFuncExp, add all parameters to the reduceKeys. + if (parseInfo.getDistinctFuncExprForClause(dest) != null) { + CommonTree value = parseInfo.getDistinctFuncExprForClause(dest); + // 0 is function name + for (int i = 1; i < value.getChildCount(); i++) { + CommonTree parameter = (CommonTree) value.getChild(i); + String text = parameter.toStringTree(); + if (reduceSinkOutputRowResolver.get("",text) == null) { + ColumnInfo exprInfo = reduceSinkInputRowResolver.get("", text); + reduceKeys.add(new exprNodeColumnDesc(exprInfo.getType(), exprInfo.getInternalName())); + reduceSinkOutputRowResolver.put("", text, + new ColumnInfo(Utilities.ReduceField.KEY.toString() + "." + Integer.valueOf(reduceKeys.size() - 1).toString(), + exprInfo.getType())); + } + } + } + + // Put partial aggregation results in reduceValues + ArrayList reduceValues = new ArrayList(); + HashMap aggregationTrees = parseInfo + .getAggregationExprsForClause(dest); + int inputField = reduceKeys.size(); + + for (Map.Entry entry : aggregationTrees.entrySet()) { + + TypeInfo type = reduceSinkInputRowResolver.getColumnInfos().get(inputField).getType(); + reduceValues.add(new exprNodeColumnDesc( + type, (Integer.valueOf(inputField)).toString())); + inputField++; + reduceSinkOutputRowResolver.put("", ((CommonTree)entry.getValue()).toStringTree(), + new ColumnInfo(Utilities.ReduceField.VALUE.toString() + "." + (Integer.valueOf(reduceValues.size()-1)).toString(), + type)); + } + + return new OperatorInfo( + OperatorFactory.getAndMakeChild( + PlanUtils.getReduceSinkDesc(reduceKeys, reduceValues, -1, + (parseInfo.getDistinctFuncExprForClause(dest) == null ? -1 : Integer.MAX_VALUE), -1, false), + new RowSchema(reduceSinkOutputRowResolver.getColumnInfos()), + inputOperatorInfo.getOp()), + reduceSinkOutputRowResolver); + } + + @SuppressWarnings("nls") + private OperatorInfo genGroupByPlanReduceSinkOperator(QB qb, String dest, OperatorInfo inputOperatorInfo, int numPartitionFields) throws SemanticException { + QBParseInfo parseInfo = qb.getParseInfo(); RowResolver reduceSinkInputRowResolver = inputOperatorInfo.getRowResolver(); RowResolver reduceSinkOutputRowResolver = new RowResolver(); reduceSinkOutputRowResolver.setIsExprResolver(true); @@ -1038,13 +1379,12 @@ List grpByExprs = getGroupByForClause(parseInfo, dest); for (int i = 0; i < grpByExprs.size(); ++i) { CommonTree grpbyExpr = grpByExprs.get(i); - reduceKeys.add(genExprNodeDesc(grpbyExpr, parseInfo.getAlias(), - reduceSinkInputRowResolver)); + reduceKeys.add(genExprNodeDesc(qb.getMetaData(), grpbyExpr, reduceSinkInputRowResolver)); String text = grpbyExpr.toStringTree(); if (reduceSinkOutputRowResolver.get("", text) == null) { reduceSinkOutputRowResolver.put("", text, new ColumnInfo(Utilities.ReduceField.KEY.toString() + "." + Integer.valueOf(reduceKeys.size() - 1).toString(), - String.class)); // Everything is a string right now + reduceKeys.get(reduceKeys.size()-1).getTypeInfo())); } else { throw new SemanticException(ErrorMsg.DUPLICATE_GROUPBY_KEY.getMsg(grpbyExpr)); } @@ -1058,10 +1398,10 @@ CommonTree parameter = (CommonTree) value.getChild(i); String text = parameter.toStringTree(); if (reduceSinkOutputRowResolver.get("",text) == null) { - reduceKeys.add(genExprNodeDesc(parameter, parseInfo.getAlias(), reduceSinkInputRowResolver)); + reduceKeys.add(genExprNodeDesc(qb.getMetaData(), parameter, reduceSinkInputRowResolver)); reduceSinkOutputRowResolver.put("", text, new ColumnInfo(Utilities.ReduceField.KEY.toString() + "." + Integer.valueOf(reduceKeys.size() - 1).toString(), - String.class)); // Everything is a string right now + reduceKeys.get(reduceKeys.size()-1).getTypeInfo())); } } } @@ -1077,10 +1417,10 @@ CommonTree parameter = (CommonTree) value.getChild(i); String text = parameter.toStringTree(); if (reduceSinkOutputRowResolver.get("",text) == null) { - reduceValues.add(genExprNodeDesc(parameter, parseInfo.getAlias(), reduceSinkInputRowResolver)); + reduceValues.add(genExprNodeDesc(qb.getMetaData(), parameter, reduceSinkInputRowResolver)); reduceSinkOutputRowResolver.put("", text, new ColumnInfo(Utilities.ReduceField.VALUE.toString() + "." + Integer.valueOf(reduceValues.size() - 1).toString(), - String.class)); // Everything is a string right now + reduceValues.get(reduceValues.size()-1).getTypeInfo())); } } } @@ -1095,9 +1435,10 @@ } @SuppressWarnings("nls") - private OperatorInfo genGroupByPlanReduceSinkOperator(QBParseInfo parseInfo, + private OperatorInfo genGroupByPlanReduceSinkOperator(QB qb, OperatorInfo input, CommonTree distinctText, TreeSet ks) throws SemanticException { + QBParseInfo parseInfo = qb.getParseInfo(); RowResolver inputRS = input.getRowResolver(); RowResolver outputRS = new RowResolver(); outputRS.setIsExprResolver(true); @@ -1106,12 +1447,13 @@ // Spray on distinctText first if (distinctText != null) { - reduceKeys.add(genExprNodeDesc(distinctText, parseInfo.getAlias(), inputRS)); + exprNodeDesc node = genExprNodeDesc(qb.getMetaData(), distinctText, inputRS); + reduceKeys.add(node); String text = distinctText.toStringTree(); assert (outputRS.get("", text) == null); outputRS.put("", text, new ColumnInfo(Utilities.ReduceField.KEY.toString() + "." + Integer.valueOf(reduceKeys.size() - 1).toString(), - String.class)); + node.getTypeInfo())); } else { // dummy key @@ -1148,7 +1490,7 @@ CommonTree grpbyExpr = grpByExprs.get(i); String text = grpbyExpr.toStringTree(); if (outputRS.get("", text) == null) { - exprNodeDesc grpbyExprNode = genExprNodeDesc(grpbyExpr, parseInfo.getAlias(), inputRS); + exprNodeDesc grpbyExprNode = genExprNodeDesc(qb.getMetaData(), grpbyExpr, inputRS); reduceValues.add(grpbyExprNode); outputRS.put("", text, new ColumnInfo(Utilities.ReduceField.VALUE.toString() + "." + Integer.valueOf(reduceValues.size() - 1).toString(), @@ -1165,7 +1507,7 @@ CommonTree parameter = (CommonTree) value.getChild(i); String text = parameter.toStringTree(); if (outputRS.get("",text) == null) { - exprNodeDesc pNode = genExprNodeDesc(parameter, parseInfo.getAlias(), inputRS); + exprNodeDesc pNode = genExprNodeDesc(qb.getMetaData(), parameter, inputRS); reduceValues.add(pNode); outputRS.put("", text, new ColumnInfo(Utilities.ReduceField.VALUE.toString() + "." + Integer.valueOf(reduceValues.size() - 1).toString(), @@ -1200,7 +1542,9 @@ @SuppressWarnings("nls") private OperatorInfo genGroupByPlanReduceSinkOperator2MR( QBParseInfo parseInfo, String dest, OperatorInfo groupByOperatorInfo, - int numPartitionFields) { + int numPartitionFields) + throws SemanticException { + RowResolver reduceSinkInputRowResolver2 = groupByOperatorInfo.getRowResolver(); RowResolver reduceSinkOutputRowResolver2 = new RowResolver(); reduceSinkOutputRowResolver2.setIsExprResolver(true); ArrayList reduceKeys = new ArrayList(); @@ -1209,10 +1553,11 @@ for (int i = 0; i < grpByExprs.size(); ++i) { CommonTree grpbyExpr = grpByExprs.get(i); String field = (Integer.valueOf(i)).toString(); - reduceKeys.add(new exprNodeColumnDesc(TypeInfoFactory.getPrimitiveTypeInfo(String.class), field)); + TypeInfo typeInfo = reduceSinkInputRowResolver2.get("", grpbyExpr.toStringTree()).getType(); + reduceKeys.add(new exprNodeColumnDesc(typeInfo, field)); reduceSinkOutputRowResolver2.put("", grpbyExpr.toStringTree(), new ColumnInfo(Utilities.ReduceField.KEY.toString() + "." + field, - String.class)); // Everything is a string right now + typeInfo)); } // Get partial aggregation results and store in reduceValues ArrayList reduceValues = new ArrayList(); @@ -1220,12 +1565,14 @@ HashMap aggregationTrees = parseInfo .getAggregationExprsForClause(dest); for (Map.Entry entry : aggregationTrees.entrySet()) { - reduceValues.add(new exprNodeColumnDesc(TypeInfoFactory.getPrimitiveTypeInfo(String.class), - (Integer.valueOf(inputField)).toString())); + String field = (Integer.valueOf(inputField)).toString(); + CommonTree t = entry.getValue(); + TypeInfo typeInfo = reduceSinkInputRowResolver2.get("", t.toStringTree()).getType(); + reduceValues.add(new exprNodeColumnDesc(typeInfo, field)); inputField++; - reduceSinkOutputRowResolver2.put("", ((CommonTree)entry.getValue()).toStringTree(), + reduceSinkOutputRowResolver2.put("", t.toStringTree(), new ColumnInfo(Utilities.ReduceField.VALUE.toString() + "." + (Integer.valueOf(reduceValues.size()-1)).toString(), - String.class)); // Everything is a string right now + typeInfo)); } return new OperatorInfo( @@ -1239,7 +1586,7 @@ @SuppressWarnings("nls") private OperatorInfo genGroupByPlanGroupByOperator2MR( - QBParseInfo parseInfo, String dest, OperatorInfo reduceSinkOperatorInfo2) + QBParseInfo parseInfo, String dest, OperatorInfo reduceSinkOperatorInfo2, groupByDesc.Mode mode) throws SemanticException { RowResolver groupByInputRowResolver2 = reduceSinkOperatorInfo2.getRowResolver(); RowResolver groupByOutputRowResolver2 = new RowResolver(); @@ -1274,20 +1621,19 @@ if (paraExprInfo == null) { throw new SemanticException(ErrorMsg.INVALID_COLUMN.getMsg(value)); } - String paraExpression = paraExprInfo.getInternalName(); assert(paraExpression != null); aggParameters.add(new exprNodeColumnDesc(paraExprInfo.getType(), paraExpression)); - aggregations.add(new aggregationDesc(aggClass, aggParameters, false)); + aggregations.add(new aggregationDesc(aggClass, aggParameters, ((mode == groupByDesc.Mode.FINAL) ? false : (value.getToken().getType() == HiveParser.TOK_FUNCTIONDI)))); groupByOutputRowResolver2.put("", value.toStringTree(), new ColumnInfo(Integer.valueOf(groupByKeys.size() + aggregations.size() - 1).toString(), - paraExprInfo.getType())); // Everything is a string right now + paraExprInfo.getType())); } return new OperatorInfo( - OperatorFactory.getAndMakeChild(new groupByDesc(groupByDesc.Mode.PARTIAL2, groupByKeys, aggregations), - new RowSchema(groupByOutputRowResolver2.getColumnInfos()), - reduceSinkOperatorInfo2.getOp()), + OperatorFactory.getAndMakeChild(new groupByDesc(mode, groupByKeys, aggregations), + new RowSchema(groupByOutputRowResolver2.getColumnInfos()), + reduceSinkOperatorInfo2.getOp()), groupByOutputRowResolver2 ); } @@ -1316,7 +1662,7 @@ // ////// 1. Generate ReduceSinkOperator OperatorInfo reduceSinkOperatorInfo = genGroupByPlanReduceSinkOperator( - parseInfo, dest, inputOperatorInfo, + qb, dest, inputOperatorInfo, getGroupByForClause(parseInfo, dest).size()); @@ -1357,7 +1703,7 @@ // operator. We set the numPartitionColumns to -1 for this purpose. This is // captured by WritableComparableHiveObject.hashCode() function. OperatorInfo reduceSinkOperatorInfo = genGroupByPlanReduceSinkOperator( - parseInfo, dest, inputOperatorInfo, (parseInfo + qb, dest, inputOperatorInfo, (parseInfo .getDistinctFuncExprForClause(dest) == null ? -1 : Integer.MAX_VALUE)); @@ -1371,8 +1717,8 @@ getGroupByForClause(parseInfo, dest).size()); // ////// 4. Generate GroupbyOperator2 - OperatorInfo groupByOperatorInfo2 = genGroupByPlanGroupByOperator2MR( - parseInfo, dest, reduceSinkOperatorInfo2); + OperatorInfo groupByOperatorInfo2 = + genGroupByPlanGroupByOperator2MR(parseInfo, dest, reduceSinkOperatorInfo2, groupByDesc.Mode.FINAL); return groupByOperatorInfo2; } @@ -1400,13 +1746,62 @@ getGroupByForClause(parseInfo, dest).size()); // ////// Generate GroupbyOperator2 - OperatorInfo groupByOperatorInfo2 = genGroupByPlanGroupByOperator2MR( - parseInfo, dest, reduceSinkOperatorInfo2); + OperatorInfo groupByOperatorInfo2 = + genGroupByPlanGroupByOperator2MR(parseInfo, dest, reduceSinkOperatorInfo2, groupByDesc.Mode.FINAL); return groupByOperatorInfo2; } + private boolean optimizeMapAggrGroupBy(String dest, QB qb) { + List grpByExprs = getGroupByForClause(qb.getParseInfo(), dest); + if ((grpByExprs != null) && !grpByExprs.isEmpty()) + return false; + + if (qb.getParseInfo().getDistinctFuncExprForClause(dest) != null) + return false; + + return true; + } + + /** + * Generate a Group-By plan using a 2 map-reduce jobs. First perform a map + * side partial aggregation (to reduce the amount of data). Then spray by + * the distinct key (or a random number) in hope of getting a uniform + * distribution, and compute partial aggregates grouped by that distinct key. + * Evaluate partial aggregates first, followed by actual aggregates. + */ @SuppressWarnings("nls") + private OperatorInfo genGroupByPlan4MR(String dest, QB qb, + OperatorInfo inputOperatorInfo) throws SemanticException { + + QBParseInfo parseInfo = qb.getParseInfo(); + + // ////// Generate GroupbyOperator for a map-side partial aggregation + OperatorInfo groupByOperatorInfo = genGroupByPlanMapGroupByOperator(qb, + dest, inputOperatorInfo, groupByDesc.Mode.HASH); + + // ////// Generate ReduceSink Operator + OperatorInfo reduceSinkOperatorInfo = + genGroupByPlanReduceSinkOperator(parseInfo, dest, groupByOperatorInfo); + + // Optimize the scenario when there are no grouping keys and no distinct - 2 map-reduce jobs are not needed + if (!optimizeMapAggrGroupBy(dest, qb)) { + // ////// Generate GroupbyOperator for a partial aggregation + OperatorInfo groupByOperatorInfo2 = genGroupByPlanGroupByOperator1(parseInfo, dest, reduceSinkOperatorInfo, + groupByDesc.Mode.PARTIAL2); + + // ////// Generate ReduceSinkOperator2 + OperatorInfo reduceSinkOperatorInfo2 = genGroupByPlanReduceSinkOperator2MR(parseInfo, dest, groupByOperatorInfo2, + getGroupByForClause(parseInfo, dest).size()); + + // ////// Generate GroupbyOperator3 + return genGroupByPlanGroupByOperator2MR(parseInfo, dest, reduceSinkOperatorInfo2, groupByDesc.Mode.FINAL); + } + else + return genGroupByPlanGroupByOperator2MR(parseInfo, dest, reduceSinkOperatorInfo, groupByDesc.Mode.FINAL); + } + + @SuppressWarnings("nls") private OperatorInfo genConversionOps(String dest, QB qb, OperatorInfo input) throws SemanticException { @@ -1471,7 +1866,6 @@ } case QBMetaData.DEST_LOCAL_FILE: case QBMetaData.DEST_DFS_FILE: { - table_desc = Utilities.defaultTd; dest_path = qb.getMetaData().getDestFileForAlias(dest); String cols = new String(); RowResolver inputRR = input.getRowResolver(); @@ -1492,12 +1886,16 @@ this.loadFileWork.add(new loadFileDesc(queryTmpdir, dest_path, (dest_type.intValue() == QBMetaData.DEST_DFS_FILE), cols)); + table_desc = PlanUtils.getDefaultTableDesc(Integer.toString(Utilities.ctrlaCode), + cols); break; } default: throw new SemanticException("Unknown destination type: " + dest_type); } + input = genConversionSelectOperator(dest, qb, input, table_desc); + OperatorInfo output = (OperatorInfo)input.clone(); output.setOp( OperatorFactory.getAndMakeChild( @@ -1512,6 +1910,84 @@ return output; } + /** + * Generate the conversion SelectOperator that converts the columns into + * the types that are expected by the table_desc. + */ + OperatorInfo genConversionSelectOperator(String dest, QB qb, + OperatorInfo input, tableDesc table_desc) throws SemanticException { + StructObjectInspector oi = null; + try { + Deserializer deserializer = table_desc.getDeserializerClass().newInstance(); + deserializer.initialize(null, table_desc.getProperties()); + oi = (StructObjectInspector) deserializer.getObjectInspector(); + } catch (Exception e) { + throw new SemanticException(e); + } + + // Check column number + List tableFields = oi.getAllStructFieldRefs(); + Vector rowFields = input.getRowResolver().getColumnInfos(); + if (tableFields.size() != rowFields.size()) { + String reason = "Table " + dest + " has " + tableFields.size() + " columns but query has " + + rowFields + "."; + throw new SemanticException(ErrorMsg.TARGET_TABLE_COLUMN_MISMATCH.getMsg( + qb.getParseInfo().getDestForClause(dest), reason)); + } + + // Check column types + boolean converted = false; + int columnNumber = tableFields.size(); + ArrayList expressions = new ArrayList(columnNumber); + // MetadataTypedColumnsetSerDe does not need type conversions because it does + // the conversion to String by itself. + if (! table_desc.getDeserializerClass().equals(MetadataTypedColumnsetSerDe.class)) { + for (int i=0; i[] rightOps = new Operator[right.length]; @@ -1674,7 +2151,7 @@ Vector exprs = joinTree.getExpressions().get(pos); for (int i = 0; i < exprs.size(); i++) { CommonTree expr = exprs.get(i); - reduceKeys.add(genExprNodeDesc(expr, srcName, inputRS)); + reduceKeys.add(genExprNodeDesc(qb.getMetaData(), expr, inputRS)); } // Walk over the input row resolver and copy in the output @@ -1765,6 +2242,14 @@ } } } + // regenerate keySerializationInfo because the ReduceSinkOperator's + // output key types might have changed. + for (int i=0; i map) @@ -1807,9 +2292,9 @@ if ((left.getToken().getType() == HiveParser.TOK_TABREF) || (left.getToken().getType() == HiveParser.TOK_SUBQUERY)) { - String table_name = left.getChild(0).getText(); - String alias = left.getChildCount() == 1 ? table_name : left.getChild(1) - .getText(); + String table_name = unescapeIdentifier(left.getChild(0).getText()); + String alias = left.getChildCount() == 1 ? table_name : + unescapeIdentifier(left.getChild(1).getText()); joinTree.setLeftAlias(alias); String[] leftAliases = new String[1]; leftAliases[0] = alias; @@ -1832,9 +2317,9 @@ if ((right.getToken().getType() == HiveParser.TOK_TABREF) || (right.getToken().getType() == HiveParser.TOK_SUBQUERY)) { - String table_name = right.getChild(0).getText(); - String alias = right.getChildCount() == 1 ? table_name : right.getChild(1) - .getText(); + String table_name = unescapeIdentifier(right.getChild(0).getText()); + String alias = right.getChildCount() == 1 ? table_name : + unescapeIdentifier(right.getChild(1).getText()); String[] rightAliases = new String[1]; rightAliases[0] = alias; joinTree.setRightAliases(rightAliases); @@ -1996,71 +2481,24 @@ TreeSet ks = new TreeSet(); ks.addAll(qbp.getClauseNames()); - String distinctText = null; - CommonTree distn = null; - OperatorInfo op = null; - boolean grpBy = false; - int numGrpBy = 0; - - // In case of a multiple group bys, all of them should have the same distinct key - for (String dest : ks) { - // is it a group by - if ((qbp.getAggregationExprsForClause(dest).size() != 0) - || (getGroupByForClause(qbp, dest).size() > 0)) { - grpBy = true; - numGrpBy++; - - // If there is a distinctFuncExp, add all parameters to the reduceKeys. - if (qbp.getDistinctFuncExprForClause(dest) != null) { - CommonTree value = qbp.getDistinctFuncExprForClause(dest); - if (value.getChildCount() != 2) - throw new SemanticException(ErrorMsg.UNSUPPORTED_MULTIPLE_DISTINCTS.getMsg(value)); - distn = (CommonTree)value.getChild(1); - String dist = distn.toStringTree();; - if (distinctText == null) - distinctText = dist; - if (!distinctText.equals(dist)) - throw new SemanticException(ErrorMsg.UNSUPPORTED_MULTIPLE_DISTINCTS.getMsg(value)); - } - } - } - - // In the first stage, copy the input and all the group by expressions - // and aggregate paramaters. This can be optimized in the future to only - // evaluate expressions that occur frequently. For a single groupby, no need to do so - if (grpBy && (numGrpBy > 1)) { - OperatorInfo reduceSinkOperatorInfo = - genGroupByPlanReduceSinkOperator(qbp, input, distn, ks); - - // ////// 2. Generate GroupbyOperator - OperatorInfo forwardOperatorInfo = genGroupByPlanForwardOperator(qbp, reduceSinkOperatorInfo); - op = forwardOperatorInfo; - } - // Go over all the destination tables OperatorInfo curr = null; for (String dest : ks) { - boolean groupByExpr = false; - if (qbp.getAggregationExprsForClause(dest).size() != 0 - || getGroupByForClause(qbp, dest).size() > 0) - groupByExpr = true; - curr = input; - if (groupByExpr && (numGrpBy > 1)) - curr = op; if (qbp.getWhrForClause(dest) != null) { curr = genFilterPlan(dest, qb, curr); } if (qbp.getAggregationExprsForClause(dest).size() != 0 - || getGroupByForClause(qbp, dest).size() > 0) { - if (numGrpBy > 1) - curr = genGroupByPlan3MR(dest, qb, curr); + || getGroupByForClause(qbp, dest).size() > 0) + { + if (conf.getVar(HiveConf.ConfVars.HIVEMAPSIDEAGGREGATE).equalsIgnoreCase("true")) + curr = genGroupByPlan4MR(dest, qb, curr); else curr = genGroupByPlan2MR(dest, qb, curr); } - + curr = genSelectPlan(dest, qb, curr); Integer limit = qbp.getDestLimit(dest); @@ -2133,8 +2571,31 @@ return unionout; } - private exprNodeDesc genSamplePredicate(TableSample ts) { - // ((default_sample_hashfn(cols) & Integer.MAX_VALUE) % denominator) == numerator + /** + * Generates the sampling predicate from the TABLESAMPLE clause information. This function uses the + * bucket column list to decide the expression inputs to the predicate hash function in case useBucketCols + * is set to true, otherwise the expression list stored in the TableSample is used. The bucket columns of + * the table are used to generate this predicate in case no expressions are provided on the TABLESAMPLE + * clause and the table has clustering columns defined in it's metadata. + * The predicate created has the following structure: + * + * ((default_sample_hashfn(expressions) & Integer.MAX_VALUE) % denominator) == numerator + * + * @param ts TABLESAMPLE clause information + * @param bucketCols The clustering columns of the table + * @param useBucketCols Flag to indicate whether the bucketCols should be used as input to the hash + * function + * @param alias The alias used for the table in the row resolver + * @param rwsch The row resolver used to resolve column references + * @param qbm The metadata information for the query block which is used to resolve unaliased columns + * @return exprNodeDesc + * @exception SemanticException + */ + private exprNodeDesc genSamplePredicate(TableSample ts, List bucketCols, + boolean useBucketCols, String alias, + RowResolver rwsch, QBMetaData qbm) + throws SemanticException { + exprNodeDesc numeratorExpr = new exprNodeConstantDesc( TypeInfoFactory.getPrimitiveTypeInfo(Integer.class), Integer.valueOf(ts.getNumerator() - 1)); @@ -2146,11 +2607,20 @@ exprNodeDesc intMaxExpr = new exprNodeConstantDesc( TypeInfoFactory.getPrimitiveTypeInfo(Integer.class), Integer.valueOf(Integer.MAX_VALUE)); + ArrayList args = new ArrayList(); - for (String col: ts.getCols()) { - // TODO: change type to the one in the table schema - args.add(new exprNodeColumnDesc(String.class, col)); + if (useBucketCols) { + for (String col : bucketCols) { + ColumnInfo ci = rwsch.get(alias, col); + // TODO: change type to the one in the table schema + args.add(new exprNodeColumnDesc(ci.getType().getPrimitiveClass(), col)); + } } + else { + for(CommonTree expr: ts.getExprs()) { + args.add(genExprNodeDesc(qbm, expr, rwsch)); + } + } exprNodeDesc hashfnExpr = getFuncExprNodeDesc("default_sample_hashfn", args); assert(hashfnExpr != null); @@ -2208,53 +2678,50 @@ if (ts != null) { int num = ts.getNumerator(); int den = ts.getDenominator(); - ArrayList sampleCols = ts.getCols(); - List tabCols = tab.getCols(); - // check if sampleCols are present in the table - for (String col: sampleCols) { - boolean found = false; - for (FieldSchema s: tabCols) { - if (col.equalsIgnoreCase(s.getName())) { - found = true; - } - } - if (!found) { - throw new SemanticException(ErrorMsg.SAMPLE_COLUMN_NOT_FOUND.getMsg( - qb.getParseInfo().getSrcForAlias(alias), "Sampling column " + - col + " not found in table " + tab.getName())); - } - } + ArrayList sampleExprs = ts.getExprs(); + + // TODO: Do the type checking of the expressions List tabBucketCols = tab.getBucketCols(); int numBuckets = tab.getNumBuckets(); + + // If there are no sample cols and no bucket cols then throw an error + if (tabBucketCols.size() == 0 && sampleExprs.size() == 0) { + throw new SemanticException(ErrorMsg.NON_BUCKETED_TABLE.getMsg() + " " + tab.getName()); + } + // check if a predicate is needed // predicate is needed if either input pruning is not enough // or if input pruning is not possible // check if the sample columns are the same as the table bucket columns - // and if they are, create a new array of column names which is in the - // same order as tabBucketCols. - // if sample cols is not specified then default is bucket cols boolean colsEqual = true; - if ( (sampleCols.size() != tabBucketCols.size()) && (sampleCols.size() != 0) ) { + if ( (sampleExprs.size() != tabBucketCols.size()) && (sampleExprs.size() != 0) ) { colsEqual = false; } - for (int i = 0; i < sampleCols.size() && colsEqual; i++) { + + for (int i = 0; i < sampleExprs.size() && colsEqual; i++) { boolean colFound = false; for (int j = 0; j < tabBucketCols.size() && !colFound; j++) { - if (sampleCols.get(i).equalsIgnoreCase(tabBucketCols.get(j))) { + if (sampleExprs.get(i).getToken().getType() != HiveParser.TOK_COLREF) { + break; + } + + if (sampleExprs.get(i).getChildCount() != 1) { + throw new SemanticException(ErrorMsg.TABLE_ALIAS_NOT_ALLOWED.getMsg()); + } + + if (((CommonTree)sampleExprs.get(i).getChild(0)).getText().equalsIgnoreCase(tabBucketCols.get(j))) { colFound = true; } } - colsEqual = colFound; + colsEqual = (colsEqual && colFound); } - // if the sample columns are the same, we need them in the same order - // as tabBucketCols - if (colsEqual) { - ts.setCols(new ArrayList(tabBucketCols)); - } + // Check if input can be pruned + ts.setInputPruning((sampleExprs == null || sampleExprs.size() == 0 || colsEqual)); + // check if input pruning is enough - if ((sampleCols == null || sampleCols.size() == 0 || colsEqual) + if ((sampleExprs == null || sampleExprs.size() == 0 || colsEqual) && (num == den || den <= numBuckets && numBuckets % den == 0)) { // input pruning is enough; no need for filter LOG.info("No need for sample filter"); @@ -2263,7 +2730,7 @@ // need to add filter // create tableOp to be filterDesc and set as child to 'top' LOG.info("Need sample filter"); - exprNodeDesc samplePredicate = genSamplePredicate(ts); + exprNodeDesc samplePredicate = genSamplePredicate(ts, tabBucketCols, colsEqual, alias, rwsch, qb.getMetaData()); tableOp = OperatorFactory.getAndMakeChild( new filterDesc(samplePredicate), top); @@ -2325,11 +2792,7 @@ OperatorInfo bodyOpInfo = genBodyPlan(qb, srcOpInfo); LOG.debug("Created Plan for Query Block " + qb.getId()); - - // is it a top level QB, and can it be optimized ? For eg: select * from T does not need a map-reduce job - QBParseInfo qbp = qb.getParseInfo(); - qbp.setCanOptTopQ(qb.isSelectStarQuery()); - + return bodyOpInfo; } @@ -2363,15 +2826,43 @@ Task mvTask = null; Task fetchTask = null; - if (qb.getParseInfo().getCanOptTopQ()) { + if (qb.isSelectStarQuery()) { Iterator> iter = qb.getMetaData().getAliasToTable().entrySet().iterator(); Table tab = ((Map.Entry)iter.next()).getValue(); - fetch = new fetchWork(tab.getPath(), tab.getDeserializer().getClass(), - tab.getInputFormatClass(), tab.getSchema(), qb.getParseInfo().getOuterQueryLimit()); - - fetchTask = TaskFactory.get(fetch, this.conf); - setFetchTask(fetchTask); - return; + if (!tab.isPartitioned()) { + if (qb.getParseInfo().getDestToWhereExpr().isEmpty()) + fetch = new fetchWork(tab.getPath(), Utilities.getTableDesc(tab), qb.getParseInfo().getOuterQueryLimit()); + } + else { + if (aliasToPruner.size() == 1) { + Iterator> iterP = aliasToPruner.entrySet().iterator(); + PartitionPruner pr = ((Map.Entry)iterP.next()).getValue(); + if (pr.containsPartitionCols()) { + List listP = new ArrayList(); + List partP = new ArrayList(); + Set parts = null; + try { + parts = pr.prune(); + Iterator iterParts = parts.iterator(); + while (iterParts.hasNext()) { + Partition part = iterParts.next(); + listP.add(part.getPartitionPath()); + partP.add(Utilities.getPartitionDesc(part)); + } + fetch = new fetchWork(listP, partP, qb.getParseInfo().getOuterQueryLimit()); + } catch (HiveException e) { + // Has to use full name to make sure it does not conflict with org.apache.commons.lang.StringUtils + LOG.error(org.apache.hadoop.util.StringUtils.stringifyException(e)); + throw new SemanticException(e.getMessage(), e); + } + } + } + } + if (fetch != null) { + fetchTask = TaskFactory.get(fetch, this.conf); + setFetchTask(fetchTask); + return; + } } // In case of a select, use a fetch task instead of a move task @@ -2379,9 +2870,14 @@ if ((!loadTableWork.isEmpty()) || (loadFileWork.size() != 1)) throw new SemanticException(ErrorMsg.GENERIC_ERROR.getMsg()); String cols = loadFileWork.get(0).getColumns(); - fetch = new fetchWork(new Path(loadFileWork.get(0).getSourceDir()), - MetadataTypedColumnsetSerDe.class, TextInputFormat.class, - Utilities.makeProperties("columns", cols), qb.getParseInfo().getOuterQueryLimit()); + + fetch = new fetchWork(new Path(loadFileWork.get(0).getSourceDir()), + new tableDesc(MetadataTypedColumnsetSerDe.class, TextInputFormat.class, + IgnoreKeyTextOutputFormat.class, + Utilities.makeProperties( + org.apache.hadoop.hive.serde.Constants.SERIALIZATION_FORMAT, "" + Utilities.ctrlaCode, + "columns", cols)), + qb.getParseInfo().getOuterQueryLimit()); fetchTask = TaskFactory.get(fetch, this.conf); setFetchTask(fetchTask); @@ -2458,10 +2954,30 @@ } } plan.getAliasToWork().put(alias_id, topOp); + setKeyAndValueDesc(plan, topOp); LOG.debug("Created Map Work for " + alias_id); } } + private void setKeyAndValueDesc(mapredWork plan, Operator topOp) { + if (topOp instanceof ReduceSinkOperator) { + ReduceSinkOperator rs = (ReduceSinkOperator)topOp; + plan.setKeyDesc(rs.getConf().getKeySerializeInfo()); + int tag = Math.max(0, rs.getConf().getTag()); + List tagToSchema = plan.getTagToValueDesc(); + while (tag + 1 > tagToSchema.size()) { + tagToSchema.add(null); + } + tagToSchema.set(tag, rs.getConf().getValueSerializeInfo()); + } else { + List> children = topOp.getChildOperators(); + if (children != null) { + for(Operator op: children) { + setKeyAndValueDesc(plan, op); + } + } + } + } @SuppressWarnings("nls") private void genTaskPlan(Operator op, Task currTask, HashMap, Task> redTaskMap, @@ -2536,8 +3052,8 @@ isfirst = false; } - tableDesc tt_desc = PlanUtils.getDefaultTableDesc(Integer.toString(Utilities.ctrlaCode), - sb.toString()); + tableDesc tt_desc = PlanUtils.getBinaryTableDesc( + PlanUtils.getFieldSchemasFromRowSchema(op.getSchema(), "temporarycol")); // Create a file sink operator for this file name Operator fs_op = OperatorFactory.get(new fileSinkDesc(taskTmpDir, tt_desc), @@ -2564,6 +3080,7 @@ new partitionDesc(tt_desc, null)); cplan.getAliasToWork().put(streamDesc, child); + setKeyAndValueDesc(cplan, child); // Make this task dependent on the current task currTask.addDependentTask(ctask); @@ -2599,6 +3116,7 @@ work.setPathToAliases(new LinkedHashMap>()); work.setPathToPartitionInfo(new LinkedHashMap()); work.setAliasToWork(new HashMap>()); + work.setTagToValueDesc(new ArrayList()); work.setReducer(null); return work; @@ -2735,8 +3253,18 @@ } + /** + * Generates and expression node descriptor for the expression passed in the arguments. This + * function uses the row resolver and the metadata informatinon that are passed as arguments + * to resolve the column names to internal names. + * @param qbm The metadata infromation for the query block + * @param expr The expression + * @param input The row resolver + * @return exprNodeDesc + * @throws SemanticException + */ @SuppressWarnings("nls") - private exprNodeDesc genExprNodeDesc(CommonTree expr, String alias, RowResolver input) + private exprNodeDesc genExprNodeDesc(QBMetaData qbm, CommonTree expr, RowResolver input) throws SemanticException { // We recursively create the exprNodeDesc. Base cases: when we encounter // a column ref, we convert that into an exprNodeColumnDesc; when we encounter @@ -2762,25 +3290,29 @@ switch (tokType) { case HiveParser.TOK_COLREF: { - // For now only allow columns of the form tab.col - if (expr.getChildCount() == 1) { - throw new SemanticException(ErrorMsg.NO_TABLE_ALIAS.getMsg(expr.getChild(0))); + String tabAlias = null; + String colName = null; + if (expr.getChildCount() != 1) { + tabAlias = unescapeIdentifier(expr.getChild(0).getText()); + colName = unescapeIdentifier(expr.getChild(1).getText()); } - - String tabAlias = expr.getChild(0).getText(); - String colName = expr.getChild(1).getText(); - if (tabAlias == null || colName == null) { + else { + colName = unescapeIdentifier(expr.getChild(0).getText()); + } + + if (colName == null) { throw new SemanticException(ErrorMsg.INVALID_XPATH.getMsg(expr)); } + colInfo = input.get(tabAlias, colName); if (colInfo == null && input.getIsExprResolver()) { throw new SemanticException(ErrorMsg.NON_KEY_EXPR_IN_GROUPBY.getMsg(expr)); } - else if (!input.hasTableAlias(tabAlias)) { + else if (tabAlias != null && !input.hasTableAlias(tabAlias)) { throw new SemanticException(ErrorMsg.INVALID_TABLE_ALIAS.getMsg(expr.getChild(0))); } else if (colInfo == null) { - throw new SemanticException(ErrorMsg.INVALID_COLUMN.getMsg(expr.getChild(1))); + throw new SemanticException(ErrorMsg.INVALID_COLUMN.getMsg(tabAlias == null? expr.getChild(0) : expr.getChild(1))); } desc = new exprNodeColumnDesc(colInfo.getType(), colInfo.getInternalName()); @@ -2794,7 +3326,7 @@ int childrenBegin = (isFunction ? 1 : 0); ArrayList children = new ArrayList(expr.getChildCount() - childrenBegin); for (int ci=childrenBegin; ci ent: qbm.getAliasToTable().entrySet()) { + for(FieldSchema field: ent.getValue().getAllCols()) { + if (colName.equalsIgnoreCase(field.getName())) { + if (found) { + throw new SemanticException(ErrorMsg.AMBIGOUS_COLUMN.getMsg(pt)); + } + + found = true; + tabAlias = ent.getKey(); + } + } + } + return tabAlias; + } } Index: src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/parse/QBParseInfo.java =================================================================== --- src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/parse/QBParseInfo.java (revision 711472) +++ src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/parse/QBParseInfo.java (working copy) @@ -32,7 +32,6 @@ public class QBParseInfo { private boolean isSubQ; - private boolean canOptTopQ; private String alias; private CommonTree joinExpr; private HashMap aliasToSrc; @@ -67,7 +66,6 @@ this.alias = alias; this.isSubQ = isSubQ; - this.canOptTopQ = false; this.outerQueryLimit = -1; } @@ -127,6 +125,10 @@ return this.destToWhereExpr.get(clause); } + public HashMap getDestToWhereExpr() { + return destToWhereExpr; + } + public CommonTree getGroupByForClause(String clause) { return this.destToGroupby.get(clause); } @@ -151,14 +153,6 @@ return this.isSubQ; } - public boolean getCanOptTopQ() { - return this.canOptTopQ; - } - - public void setCanOptTopQ(boolean canOptTopQ) { - this.canOptTopQ = canOptTopQ; - } - public CommonTree getJoinExpr() { return this.joinExpr; } @@ -201,7 +195,6 @@ if (isSubQ || (joinExpr != null) || (!nameToSample.isEmpty()) || - (!destToWhereExpr.isEmpty()) || (!destToGroupby.isEmpty()) || (!destToClusterby.isEmpty())) return false; Index: src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/parse/SamplePruner.java =================================================================== --- src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/parse/SamplePruner.java (revision 711472) +++ src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/parse/SamplePruner.java (working copy) @@ -22,43 +22,96 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hive.ql.metadata.Partition; import org.apache.hadoop.fs.Path; -import java.util.*; +/** + * + * This class stores the mapping from table alias to the parse tree information of the table + * sample clause(stored in the TableSample class). + * + */ +public class SamplePruner { -public class SamplePruner { + /** + * Table alias for the table e.g. in case of FROM t TABLESAMPLE(1 OUT OF 2 ON rand()) a + * "a" is the table alias + */ private String tabAlias; + + /** + * The parse tree corresponding to the TABLESAMPLE clause. e.g. in case of + * FROM t TABLESAMPLE(1 OUT OF 2 ON rand()) a the parse tree of + * "TABLESAMPLE(1 OUT OF 2 ON rand())" is parsed out and stored in tableSample + */ private TableSample tableSample; - // The log - @SuppressWarnings("nls") - private static final Log LOG = LogFactory.getLog("hive.ql.parse.SamplePruner"); + + /** + * The log handle for this class + */ + @SuppressWarnings("nls") + private static final Log LOG = LogFactory.getLog("hive.ql.parse.SamplePruner"); + /** + * Constructs the SamplePruner given the table alias and the table sample + * + * @param alias The alias of the table specified in the query + * @param tableSample The parse infromation of the TABLESAMPLE clause + */ public SamplePruner(String alias, TableSample tableSample) { this.tabAlias = alias; this.tableSample = tableSample; } + + /** + * Gets the table alias + * + * @return String + */ public String getTabAlias() { return this.tabAlias; } + + /** + * Sets the table alias + * + * @param tabAlias The table alias as specified in the query + */ public void setTabAlias(String tabAlias) { this.tabAlias = tabAlias; } + + /** + * Gets the parse information of the associated table sample clause + * + * @return TableSample + */ public TableSample getTableSample() { return this.tableSample; } + + /** + * Sets the parse information of the associated table sample clause + * + * @param tableSample Information related to the table sample clause + */ public void setTableSample(TableSample tableSample) { this.tableSample = tableSample; } + /** + * Prunes to get all the files in the partition that satisfy the TABLESAMPLE clause + * + * @param part The partition to prune + * @return Path[] + * @throws SemanticException + */ @SuppressWarnings("nls") public Path[] prune(Partition part) throws SemanticException { int num = this.tableSample.getNumerator(); int den = this.tableSample.getDenominator(); int bucketCount = part.getBucketCount(); - List tabBucketCols = part.getBucketCols(); - ArrayList sampleCols = this.tableSample.getCols(); String fullScanMsg = ""; // check if input pruning is possible - if (sampleCols == null || sampleCols.size() == 0 || tabBucketCols.equals(sampleCols)) { + if (this.tableSample.getInputPruning()) { LOG.trace("numerator = " + num); LOG.trace("denominator = " + den); LOG.trace("bucket count = " + bucketCount); Index: src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/parse/RowResolver.java =================================================================== --- src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/parse/RowResolver.java (revision 711472) +++ src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/parse/RowResolver.java (working copy) @@ -77,14 +77,49 @@ return rslvMap.get(tab_alias.toLowerCase()) != null; } - public ColumnInfo get(String tab_alias, String col_alias) { - tab_alias = tab_alias.toLowerCase(); + /** + * Gets the column Info to tab_alias.col_alias type of a column reference. I the tab_alias is not + * provided as can be the case with an non aliased column, this function looks up the column in all + * the table aliases in this row resolver and returns the match. It also throws an exception if + * the column is found in multiple table aliases. If no match is found a null values is returned. + * + * This allows us to interpret both select t.c1 type of references and select c1 kind of refereneces. + * The later kind are what we call non aliased column references in the query. + * + * @param tab_alias The table alias to match (this is null if the column reference is non aliased) + * @param col_alias The column name that is being searched for + * @return ColumnInfo + * @throws SemanticException + */ + public ColumnInfo get(String tab_alias, String col_alias) + throws SemanticException { col_alias = col_alias.toLowerCase(); - HashMap f_map = rslvMap.get(tab_alias); - if (f_map == null) { - return null; + ColumnInfo ret = null; + + if (tab_alias != null) { + tab_alias = tab_alias.toLowerCase(); + HashMap f_map = rslvMap.get(tab_alias); + if (f_map == null) { + return null; + } + ret = f_map.get(col_alias); } - return f_map.get(col_alias); + else { + boolean found = false; + for(LinkedHashMap cmap: rslvMap.values()) { + for(Map.Entry cmapEnt: cmap.entrySet()) { + if (col_alias.equalsIgnoreCase((String)cmapEnt.getKey())) { + if (found) { + throw new SemanticException("Column " + col_alias + " Found in more than One Tables/Subqueries"); + } + found = true; + ret = (ColumnInfo)cmapEnt.getValue(); + } + } + } + } + + return ret; } public Vector getColumnInfos() { Index: src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/parse/TableSample.java =================================================================== --- src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/parse/TableSample.java (revision 711472) +++ src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/parse/TableSample.java (working copy) @@ -19,34 +19,127 @@ package org.apache.hadoop.hive.ql.parse; import java.util.ArrayList; +import org.antlr.runtime.tree.CommonTree; +/** + * + * This class stores all the information specified in the TABLESAMPLE clause. e.g. + * for the clause "FROM t TABLESAMPLE(1 OUT OF 2 ON c1) it will store the numerator + * 1, the denominator 2 and the list of expressions(in this case c1) in the appropriate + * fields. The afore-mentioned sampling clause causes the 1st bucket to be picked out of + * the 2 buckets created by hashing on c1. + * + */ public class TableSample { + + /** + * The numerator of the TABLESAMPLE clause + */ private int numerator; + + /** + * The denominator of the TABLESAMPLE clause + */ private int denominator; - private ArrayList cols; - public TableSample(String num, String den, ArrayList cols) { + /** + * The list of expressions following ON part of the TABLESAMPLE clause. This list is + * empty in case there are no expressions such as in the clause + * "FROM t TABLESAMPLE(1 OUT OF 2)". For this expression the sampling is done + * on the tables clustering column(as specified when the table was created). In case + * the table does not have any clustering column, the usage of a table sample clause + * without an ON part is disallowed by the compiler + */ + private ArrayList exprs; + + /** + * Flag to indicate that input files can be pruned + */ + private boolean inputPruning; + + /** + * Constructs the TableSample given the numerator, denominator and the list of + * ON clause expressions + * + * @param num The numerator + * @param den The denominator + * @param exprs The list of expressions in the ON part of the TABLESAMPLE clause + */ + public TableSample(String num, String den, ArrayList exprs) { this.numerator = Integer.valueOf(num).intValue(); this.denominator = Integer.valueOf(den).intValue(); - this.cols = cols; + this.exprs = exprs; } + + /** + * Gets the numerator + * + * @return int + */ public int getNumerator() { return this.numerator; } + + /** + * Sets the numerator + * + * @param num The numerator + */ public void setNumerator(int num) { this.numerator = num; } + + /** + * Gets the denominator + * + * @return int + */ public int getDenominator() { return this.denominator; } + + /** + * Sets the denominator + * + * @param den The denominator + */ public void setDenominator(int den) { this.denominator = den; } - public ArrayList getCols() { - return this.cols; + + /** + * Gets the ON part's expression list + * + * @return ArrayList + */ + public ArrayList getExprs() { + return this.exprs; } - public void setCols(ArrayList cols) { - this.cols = cols; + + /** + * Sets the expression list + * + * @param exprs The expression list + */ + public void setExprs(ArrayList exprs) { + this.exprs = exprs; } + /** + * Gets the flag that indicates whether input pruning is possible + * + * @return boolean + */ + public boolean getInputPruning() { + return this.inputPruning; + } + + /** + * Sets the flag that indicates whether input pruning is possible or not + * + * @param inputPruning true if input pruning is possible + */ + public void setInputPruning(boolean inputPruning) { + this.inputPruning = inputPruning; + } } Index: src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java =================================================================== --- src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java (revision 711472) +++ src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java (working copy) @@ -27,6 +27,7 @@ import org.antlr.runtime.tree.CommonTree; import org.antlr.runtime.tree.Tree; +import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.fs.Path; @@ -45,12 +46,15 @@ import org.apache.hadoop.hive.ql.plan.showTablesDesc; import org.apache.hadoop.hive.ql.plan.alterTableDesc.alterTableTypes; import org.apache.hadoop.hive.serde.Constants; +import org.apache.hadoop.hive.serde2.SerDeUtils; public class DDLSemanticAnalyzer extends BaseSemanticAnalyzer { private static final Log LOG = LogFactory.getLog("hive.ql.parse.DDLSemanticAnalyzer"); public static final Map TokenToTypeName = new HashMap(); static { + TokenToTypeName.put(HiveParser.TOK_BOOLEAN, Constants.BOOLEAN_TYPE_NAME); TokenToTypeName.put(HiveParser.TOK_TINYINT, Constants.TINYINT_TYPE_NAME); + TokenToTypeName.put(HiveParser.TOK_SMALLINT, Constants.SMALLINT_TYPE_NAME); TokenToTypeName.put(HiveParser.TOK_INT, Constants.INT_TYPE_NAME); TokenToTypeName.put(HiveParser.TOK_BIGINT, Constants.BIGINT_TYPE_NAME); TokenToTypeName.put(HiveParser.TOK_FLOAT, Constants.FLOAT_TYPE_NAME); @@ -96,18 +100,26 @@ analyzeAlterTableModifyCols(ast, alterTableTypes.REPLACECOLS); else if (ast.getToken().getType() == HiveParser.TOK_ALTERTABLE_DROPPARTS) analyzeAlterTableDropParts(ast); + else if (ast.getToken().getType() == HiveParser.TOK_ALTERTABLE_PROPERTIES) + analyzeAlterTableProps(ast); + else if (ast.getToken().getType() == HiveParser.TOK_ALTERTABLE_SERDEPROPERTIES) + analyzeAlterTableSerdeProps(ast); + else if (ast.getToken().getType() == HiveParser.TOK_ALTERTABLE_SERIALIZER) + analyzeAlterTableSerde(ast); else if (ast.getToken().getType() == HiveParser.TOK_SHOWPARTITIONS) { ctx.setResFile(new Path(getTmpFileName())); analyzeShowPartitions(ast); } + else { + throw new SemanticException("Unsupported command."); + } } private void analyzeCreateTable(CommonTree ast, boolean isExt) throws SemanticException { - String tableName = ast.getChild(0).getText(); - CommonTree colList = (CommonTree)ast.getChild(1); - List cols = getColumns(colList); + String tableName = unescapeIdentifier(ast.getChild(0).getText()); + List cols = null; List partCols = null; List bucketCols = null; List sortCols = null; @@ -117,19 +129,23 @@ String mapKeyDelim = null; String lineDelim = null; String comment = null; - boolean isSequenceFile = false; + boolean isSequenceFile = + "SequenceFile".equalsIgnoreCase(conf.getVar(HiveConf.ConfVars.HIVEDEFAULTFILEFORMAT)); String location = null; String serde = null; Map mapProp = null; LOG.info("Creating table" + tableName); int numCh = ast.getChildCount(); - for (int num = 2; num < numCh; num++) + for (int num = 1; num < numCh; num++) { CommonTree child = (CommonTree)ast.getChild(num); switch (child.getToken().getType()) { + case HiveParser.TOK_TABCOLLIST: + cols = getColumns(child); + break; case HiveParser.TOK_TABLECOMMENT: - comment = child.getChild(0).getText(); + comment = unescapeSQLString(child.getChild(0).getText()); break; case HiveParser.TOK_TABLEPARTCOLS: partCols = getColumns((CommonTree)child.getChild(0)); @@ -181,6 +197,9 @@ case HiveParser.TOK_TBLSEQUENCEFILE: isSequenceFile = true; break; + case HiveParser.TOK_TBLTEXTFILE: + isSequenceFile = false; + break; case HiveParser.TOK_TABLELOCATION: location = unescapeSQLString(child.getChild(0).getText()); break; @@ -203,6 +222,15 @@ // no duplicate column names // currently, it is a simple n*n algorithm - this can be optimized later if need be // but it should not be a major bottleneck as the number of columns are anyway not so big + + if((crtTblDesc.getCols() == null) || (crtTblDesc.getCols().size() == 0)) { + // for now make sure that serde exists + if(StringUtils.isEmpty(crtTblDesc.getSerName()) || SerDeUtils.isNativeSerDe(crtTblDesc.getSerName())) { + throw new SemanticException(ErrorMsg.INVALID_TBL_DDL_SERDE.getMsg()); + } + return; + } + Iterator iterCols = crtTblDesc.getCols().iterator(); List colNames = new ArrayList(); while (iterCols.hasNext()) { @@ -264,9 +292,9 @@ String partCol = partColsIter.next().getName(); Iterator colNamesIter = colNames.iterator(); while (colNamesIter.hasNext()) { - String colName = colNamesIter.next(); + String colName = unescapeIdentifier(colNamesIter.next()); if (partCol.equalsIgnoreCase(colName)) - throw new SemanticException(ErrorMsg.COLUMN_REPAEATED_IN_PARTITIONING_COLS.getMsg()); + throw new SemanticException(ErrorMsg.COLUMN_REPEATED_IN_PARTITIONING_COLS.getMsg()); } } } @@ -274,11 +302,52 @@ private void analyzeDropTable(CommonTree ast) throws SemanticException { - String tableName = ast.getChild(0).getText(); + String tableName = unescapeIdentifier(ast.getChild(0).getText()); dropTableDesc dropTblDesc = new dropTableDesc(tableName); rootTasks.add(TaskFactory.get(new DDLWork(dropTblDesc), conf)); } + private void analyzeAlterTableProps(CommonTree ast) throws SemanticException { + String tableName = unescapeIdentifier(ast.getChild(0).getText()); + HashMap mapProp = getProps((CommonTree)(ast.getChild(1)).getChild(0)); + alterTableDesc alterTblDesc = new alterTableDesc(alterTableTypes.ADDPROPS); + alterTblDesc.setProps(mapProp); + alterTblDesc.setOldName(tableName); + rootTasks.add(TaskFactory.get(new DDLWork(alterTblDesc), conf)); + } + + private void analyzeAlterTableSerdeProps(CommonTree ast) throws SemanticException { + String tableName = unescapeIdentifier(ast.getChild(0).getText()); + HashMap mapProp = getProps((CommonTree)(ast.getChild(1)).getChild(0)); + alterTableDesc alterTblDesc = new alterTableDesc(alterTableTypes.ADDSERDEPROPS); + alterTblDesc.setProps(mapProp); + alterTblDesc.setOldName(tableName); + rootTasks.add(TaskFactory.get(new DDLWork(alterTblDesc), conf)); + } + + private void analyzeAlterTableSerde(CommonTree ast) throws SemanticException { + String tableName = unescapeIdentifier(ast.getChild(0).getText()); + String serdeName = unescapeSQLString(ast.getChild(1).getText()); + alterTableDesc alterTblDesc = new alterTableDesc(alterTableTypes.ADDSERDE); + if(ast.getChildCount() > 2) { + HashMap mapProp = getProps((CommonTree)(ast.getChild(2)).getChild(0)); + alterTblDesc.setProps(mapProp); + } + alterTblDesc.setOldName(tableName); + alterTblDesc.setSerdeName(serdeName); + rootTasks.add(TaskFactory.get(new DDLWork(alterTblDesc), conf)); + } + + private HashMap getProps(CommonTree prop) { + HashMap mapProp = new HashMap(); + for (int propChild = 0; propChild < prop.getChildCount(); propChild++) { + String key = unescapeSQLString(prop.getChild(propChild).getChild(0).getText()); + String value = unescapeSQLString(prop.getChild(propChild).getChild(1).getText()); + mapProp.put(key,value); + } + return mapProp; + } + private List getColumns(CommonTree ast) { List colList = new ArrayList(); @@ -286,7 +355,7 @@ for (int i = 0; i < numCh; i++) { FieldSchema col = new FieldSchema(); CommonTree child = (CommonTree)ast.getChild(i); - col.setName(child.getChild(0).getText()); + col.setName(unescapeIdentifier(child.getChild(0).getText())); CommonTree typeChild = (CommonTree)(child.getChild(1)); if (typeChild.getToken().getType() == HiveParser.TOK_LIST) { @@ -303,7 +372,7 @@ col.setType(getTypeName(typeChild.getToken().getType())); if (child.getChildCount() == 3) - col.setComment(child.getChild(2).getText()); + col.setComment(unescapeSQLString(child.getChild(2).getText())); colList.add(col); } return colList; @@ -315,7 +384,7 @@ int numCh = ast.getChildCount(); for (int i = 0; i < numCh; i++) { CommonTree child = (CommonTree)ast.getChild(i); - colList.add(child.getText()); + colList.add(unescapeIdentifier(child.getText())); } return colList; } @@ -327,9 +396,9 @@ for (int i = 0; i < numCh; i++) { CommonTree child = (CommonTree)ast.getChild(i); if (child.getToken().getType() == HiveParser.TOK_TABSORTCOLNAMEASC) - colList.add(new Order(child.getChild(0).getText(), 1)); + colList.add(new Order(unescapeIdentifier(child.getChild(0).getText()), 1)); else - colList.add(new Order(child.getChild(0).getText(), 0)); + colList.add(new Order(unescapeIdentifier(child.getChild(0).getText()), 0)); } return colList; } @@ -359,7 +428,7 @@ private void analyzeShowPartitions(CommonTree ast) throws SemanticException { showPartitionsDesc showPartsDesc; - String tableName = ast.getChild(0).getText(); + String tableName = unescapeIdentifier(ast.getChild(0).getText()); showPartsDesc = new showPartitionsDesc(tableName, ctx.getResFile()); rootTasks.add(TaskFactory.get(new DDLWork(showPartsDesc), conf)); } @@ -379,13 +448,15 @@ private void analyzeAlterTableRename(CommonTree ast) throws SemanticException { - alterTableDesc alterTblDesc = new alterTableDesc(ast.getChild(0).getText(), ast.getChild(1).getText()); + alterTableDesc alterTblDesc = new alterTableDesc( + unescapeIdentifier(ast.getChild(0).getText()), + unescapeIdentifier(ast.getChild(1).getText())); rootTasks.add(TaskFactory.get(new DDLWork(alterTblDesc), conf)); } private void analyzeAlterTableModifyCols(CommonTree ast, alterTableTypes alterType) throws SemanticException { - String tblName = ast.getChild(0).getText(); + String tblName = unescapeIdentifier(ast.getChild(0).getText()); List newCols = getColumns((CommonTree)ast.getChild(1)); alterTableDesc alterTblDesc = new alterTableDesc(tblName, newCols, alterType); rootTasks.add(TaskFactory.get(new DDLWork(alterTblDesc), conf)); @@ -396,7 +467,7 @@ List> partSpecs = new ArrayList>(); int childIndex = 0; // get table metadata - tblName = ast.getChild(0).getText(); + tblName = unescapeIdentifier(ast.getChild(0).getText()); // get partition metadata if partition specified for( childIndex = 1; childIndex < ast.getChildCount(); childIndex++) { CommonTree partspec = (CommonTree) ast.getChild(childIndex); Index: src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/plan/mapredWork.java =================================================================== --- src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/plan/mapredWork.java (revision 711472) +++ src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/plan/mapredWork.java (working copy) @@ -39,10 +39,10 @@ // map<->reduce interface // schema of the map-reduce 'key' object - this is homogeneous - private schemaDesc keySchema; + private tableDesc keyDesc; // schema of the map-reduce 'val' object - this is heterogeneous - private HashMap aliasToSchema; + private List tagToValueDesc; private Operator reducer; @@ -57,16 +57,16 @@ final LinkedHashMap> pathToAliases, final LinkedHashMap pathToPartitionInfo, final HashMap> aliasToWork, - final schemaDesc keySchema, - HashMap aliasToSchema, + final tableDesc keyDesc, + List tagToValueDesc, final Operator reducer, final Integer numReduceTasks) { this.command = command; this.pathToAliases = pathToAliases; this.pathToPartitionInfo = pathToPartitionInfo; this.aliasToWork = aliasToWork; - this.keySchema = keySchema; - this.aliasToSchema = aliasToSchema; + this.keyDesc = keyDesc; + this.tagToValueDesc = tagToValueDesc; this.reducer = reducer; this.numReduceTasks = numReduceTasks; } @@ -100,17 +100,17 @@ public void setAliasToWork(final HashMap> aliasToWork) { this.aliasToWork=aliasToWork; } - public schemaDesc getKeySchema() { - return this.keySchema; + public tableDesc getKeyDesc() { + return this.keyDesc; } - public void setKeySchema(final schemaDesc keySchema) { - this.keySchema = keySchema; + public void setKeyDesc(final tableDesc keyDesc) { + this.keyDesc = keyDesc; } - public HashMap getAliasToSchema() { - return this.aliasToSchema; + public List getTagToValueDesc() { + return tagToValueDesc; } - public void setAliasToSchema(final HashMap aliasToSchema) { - this.aliasToSchema = aliasToSchema; + public void setTagToValueDesc(final List tagToValueDesc) { + this.tagToValueDesc = tagToValueDesc; } @explain(displayName="Reduce Operator Tree") Index: src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/plan/alterTableDesc.java =================================================================== --- src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/plan/alterTableDesc.java (revision 711472) +++ src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/plan/alterTableDesc.java (working copy) @@ -20,6 +20,7 @@ import java.io.Serializable; import java.util.List; +import java.util.Map; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.ql.exec.Utilities; @@ -28,12 +29,14 @@ public class alterTableDesc extends ddlDesc implements Serializable { private static final long serialVersionUID = 1L; - public static enum alterTableTypes {RENAME, ADDCOLS, REPLACECOLS}; + public static enum alterTableTypes {RENAME, ADDCOLS, REPLACECOLS, ADDPROPS, ADDSERDE, ADDSERDEPROPS}; alterTableTypes op; String oldName; String newName; List newCols; + String serdeName; + Map props; /** * @param oldName old name of the table @@ -54,6 +57,13 @@ this.oldName = name; this.newCols = newCols; } + + /** + * @param alterType type of alter op + */ + public alterTableDesc(alterTableTypes alterType) { + this.op = alterType; + } /** * @return the old name of the table @@ -130,4 +140,34 @@ this.newCols = newCols; } + /** + * @return the serdeName + */ + @explain(displayName="deserializer library") + public String getSerdeName() { + return serdeName; + } + + /** + * @param serdeName the serdeName to set + */ + public void setSerdeName(String serdeName) { + this.serdeName = serdeName; + } + + /** + * @return the props + */ + @explain(displayName="properties") + public Map getProps() { + return props; + } + + /** + * @param props the props to set + */ + public void setProps(Map props) { + this.props = props; + } + } Index: src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/plan/groupByDesc.java =================================================================== --- src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/plan/groupByDesc.java (revision 711472) +++ src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/plan/groupByDesc.java (working copy) @@ -23,11 +23,12 @@ /** Group-by Mode: * COMPLETE: complete 1-phase aggregation: aggregate, evaluate * PARTIAL1: partial aggregation - first phase: aggregate, evaluatePartial - * PARTIAL2: partial aggregation - second phase: aggregatePartial, evaluate + * PARTIAL2: partial aggregation - second phase: aggregatePartial, evaluatePartial + * FINAL: partial aggregation - final phase: aggregatePartial, evaluate * HASH: the same as PARTIAL1 but use hash-table-based aggregation */ private static final long serialVersionUID = 1L; - public static enum Mode { COMPLETE, PARTIAL1, PARTIAL2, HASH }; + public static enum Mode { COMPLETE, PARTIAL1, PARTIAL2, FINAL, HASH }; private Mode mode; private java.util.ArrayList keys; private java.util.ArrayList aggregators; Index: src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java =================================================================== --- src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java (revision 711472) +++ src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java (working copy) @@ -20,13 +20,24 @@ import java.util.*; import java.io.*; + +import org.apache.hadoop.hive.metastore.MetaStoreUtils; +import org.apache.hadoop.hive.metastore.api.FieldSchema; +import org.apache.hadoop.hive.ql.exec.ColumnInfo; import org.apache.hadoop.hive.ql.exec.Operator; +import org.apache.hadoop.hive.ql.exec.RowSchema; import org.apache.hadoop.hive.ql.exec.Utilities; import org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat; +import org.apache.hadoop.hive.ql.typeinfo.TypeInfoUtils; import org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; +import org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe; +import org.apache.hadoop.hive.serde2.thrift.TBinarySortableProtocol; +import org.apache.hadoop.mapred.SequenceFileInputFormat; +import org.apache.hadoop.mapred.SequenceFileOutputFormat; import org.apache.hadoop.mapred.TextInputFormat; +import com.facebook.thrift.protocol.TBinaryProtocol; + public class PlanUtils { public static enum ExpressionTypes {FIELD, JEXL}; @@ -37,8 +48,8 @@ new LinkedHashMap> (), new LinkedHashMap (), new HashMap> (), - new schemaDesc(), - new HashMap (), + new tableDesc(), + new ArrayList (), null, Integer.valueOf (1)); } @@ -52,6 +63,10 @@ org.apache.hadoop.hive.serde.Constants.SERIALIZATION_FORMAT, separatorCode, "columns", columns)); } + + /** + * Generate the table descriptor of MetadataTypedColumnsetSerDe with the separatorCode + */ public static tableDesc getDefaultTableDesc(String separatorCode) { return new tableDesc( MetadataTypedColumnsetSerDe.class, @@ -61,26 +76,86 @@ org.apache.hadoop.hive.serde.Constants.SERIALIZATION_FORMAT, separatorCode)); } + /** + * Generate the table descriptor of DynamicSerDe and TBinarySortableProtocol. + */ + public static tableDesc getBinarySortableTableDesc(List fieldSchemas) { + String structName = "binary_sortable_table"; + return new tableDesc( + DynamicSerDe.class, + SequenceFileInputFormat.class, + SequenceFileOutputFormat.class, + Utilities.makeProperties( + "name", structName, + org.apache.hadoop.hive.serde.Constants.SERIALIZATION_FORMAT, TBinarySortableProtocol.class.getName(), + org.apache.hadoop.hive.serde.Constants.SERIALIZATION_DDL, + MetaStoreUtils.getDDLFromFieldSchema(structName, fieldSchemas) + )); + } + + /** + * Generate the table descriptor of DynamicSerDe and TBinaryProtocol. + */ + public static tableDesc getBinaryTableDesc(List fieldSchemas) { + String structName = "binary_table"; + return new tableDesc( + DynamicSerDe.class, + SequenceFileInputFormat.class, + SequenceFileOutputFormat.class, + Utilities.makeProperties( + "name", structName, + org.apache.hadoop.hive.serde.Constants.SERIALIZATION_FORMAT, TBinaryProtocol.class.getName(), + org.apache.hadoop.hive.serde.Constants.SERIALIZATION_DDL, + MetaStoreUtils.getDDLFromFieldSchema(structName, fieldSchemas) + )); + } + + /** + * Convert the ColumnList to FieldSchema list. + */ + public static List getFieldSchemasFromColumnList(ArrayList cols, + String fieldPrefix) { + List schemas = new ArrayList(cols.size()); + for (int i=0; i getFieldSchemasFromRowSchema(RowSchema row, String fieldPrefix) { + Vector c = row.getSignature(); + return getFieldSchemasFromColumnInfo(c, fieldPrefix); + } + + /** + * Convert the ColumnInfo to FieldSchema. + */ + public static List getFieldSchemasFromColumnInfo(Vector cols, String fieldPrefix) { + List schemas = new ArrayList(cols.size()); + for (int i=0; i keyCols, ArrayList valueCols, int tag, int numPartitionFields, int numReducers, boolean inferNumReducers) { - + return new reduceSinkDesc(keyCols, valueCols, tag, numPartitionFields, numReducers, inferNumReducers, - getDefaultTableDesc("" + Utilities.ctrlaCode, ObjectInspectorUtils.getIntegerCSV(keyCols.size())), - getDefaultTableDesc("" + Utilities.ctrlaCode, ObjectInspectorUtils.getIntegerCSV(valueCols.size()))); + getBinarySortableTableDesc(getFieldSchemasFromColumnList(keyCols, "reducesinkkey")), + getBinaryTableDesc(getFieldSchemasFromColumnList(valueCols, "reducesinkvalue"))); } - // We should read the TableDesc from gWork when it is available. - public static tableDesc getReduceKeyDesc(mapredWork gWork) { - return getDefaultTableDesc("" + Utilities.ctrlaCode); - } - - // We should read the TableDesc from gWork when it is available. - public static tableDesc getReduceValueDesc(mapredWork gWork, int tag) { - return getDefaultTableDesc("" + Utilities.ctrlaCode); - } } Index: src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/plan/fetchWork.java =================================================================== --- src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/plan/fetchWork.java (revision 711472) +++ src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/plan/fetchWork.java (working copy) @@ -19,99 +19,91 @@ package org.apache.hadoop.hive.ql.plan; import java.io.Serializable; -import java.util.Properties; +import java.util.List; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hive.serde2.Deserializer; -import org.apache.hadoop.mapred.InputFormat; +import org.apache.hadoop.hive.ql.plan.tableDesc; @explain(displayName="Fetch Operator") public class fetchWork implements Serializable { private static final long serialVersionUID = 1L; - // private loadFileDesc loadFileWork; - // private tableDesc tblDesc; - private Path srcDir; - private Properties schema; - private Class deserializerClass; - private Class inputFormatClass; + private Path tblDir; + private tableDesc tblDesc; + + private List partDir; + private List partDesc; + private int limit; public fetchWork() { } - /** - * @param deserializer - * @param deserializerClass - * @param inputFormatClass - * @param schema - * @param srcDir - */ - public fetchWork(Path srcDir, - Class deserializerClass, - Class inputFormatClass, Properties schema, - int limit) { - this.srcDir = srcDir; - this.deserializerClass = deserializerClass; - this.inputFormatClass = inputFormatClass; - this.schema = schema; + public fetchWork(Path tblDir, tableDesc tblDesc, int limit) { + this.tblDir = tblDir; + this.tblDesc = tblDesc; this.limit = limit; } + public fetchWork(List partDir, List partDesc, int limit) { + this.partDir = partDir; + this.partDesc = partDesc; + this.limit = limit; + } + /** - * @return the srcDir + * @return the tblDir */ - @explain(displayName="source") - public Path getSrcDir() { - return srcDir; + public Path getTblDir() { + return tblDir; } /** - * @param srcDir the srcDir to set + * @param tblDir the tblDir to set */ - public void setSrcDir(Path srcDir) { - this.srcDir = srcDir; + public void setTblDir(Path tblDir) { + this.tblDir = tblDir; } /** - * @return the schema + * @return the tblDesc */ - public Properties getSchema() { - return schema; + public tableDesc getTblDesc() { + return tblDesc; } /** - * @param schema the schema to set + * @param tblDesc the tblDesc to set */ - public void setSchema(Properties schema) { - this.schema = schema; + public void setTblDesc(tableDesc tblDesc) { + this.tblDesc = tblDesc; } /** - * @return the deserializerClass + * @return the partDir */ - public Class getDeserializerClass() { - return deserializerClass; + public List getPartDir() { + return partDir; } /** - * @param deserializerClass the deserializerClass to set + * @param partDir the partDir to set */ - public void setDeserializerClass(Class deserializerClass) { - this.deserializerClass = deserializerClass; + public void setPartDir(List partDir) { + this.partDir = partDir; } /** - * @return the inputFormatClass + * @return the partDesc */ - public Class getInputFormatClass() { - return inputFormatClass; + public List getPartDesc() { + return partDesc; } /** - * @param inputFormatClass the inputFormatClass to set + * @param partDesc the partDesc to set */ - public void setInputFormatClass(Class inputFormatClass) { - this.inputFormatClass = inputFormatClass; + public void setPartDesc(List partDesc) { + this.partDesc = partDesc; } /** Index: src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/Driver.java =================================================================== --- src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/Driver.java (revision 711472) +++ src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/Driver.java (working copy) @@ -218,7 +218,7 @@ public boolean getResults(Vector res) { - if (sem.getFetchTask() != null) { + if (sem != null && sem.getFetchTask() != null) { if (!sem.getFetchTaskInit()) { sem.setFetchTaskInit(true); sem.getFetchTask().initialize(conf); Index: src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/typeinfo/PrimitiveTypeInfo.java =================================================================== --- src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/typeinfo/PrimitiveTypeInfo.java (revision 711472) +++ src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/typeinfo/PrimitiveTypeInfo.java (working copy) @@ -43,7 +43,7 @@ public PrimitiveTypeInfo() {} public String getTypeName() { - return ObjectInspectorUtils.getClassShortName(primitiveClass.getName()); + return ObjectInspectorUtils.getClassShortName(primitiveClass); } Index: src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/typeinfo/TypeInfoUtils.java =================================================================== --- src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/typeinfo/TypeInfoUtils.java (revision 711472) +++ src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/typeinfo/TypeInfoUtils.java (working copy) @@ -3,11 +3,16 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; +import java.util.Map; +import org.apache.hadoop.hive.metastore.api.FieldSchema; +import org.apache.hadoop.hive.ql.parse.HiveParser; +import org.apache.hadoop.hive.serde.Constants; import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.StructField; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; @@ -109,4 +114,67 @@ return result; } + public static String getFieldSchemaTypeFromTypeInfo(TypeInfo typeInfo) { + switch(typeInfo.getCategory()) { + case PRIMITIVE: { + return ObjectInspectorUtils.getClassShortName(typeInfo.getPrimitiveClass()); + } + case LIST: { + String elementType = getFieldSchemaTypeFromTypeInfo(typeInfo.getListElementTypeInfo()); + return org.apache.hadoop.hive.serde.Constants.LIST_TYPE_NAME + "<" + elementType + ">"; + } + case MAP: { + String keyType = getFieldSchemaTypeFromTypeInfo(typeInfo.getMapKeyTypeInfo()); + String valueType = getFieldSchemaTypeFromTypeInfo(typeInfo.getMapValueTypeInfo()); + return org.apache.hadoop.hive.serde.Constants.MAP_TYPE_NAME + "<" + + keyType + "," + valueType + ">"; + } + case STRUCT: { + throw new RuntimeException("Complex struct type not supported!"); + } + default: { + throw new RuntimeException("Unknown type!"); + } + } + } + + /** + * Convert TypeInfo to FieldSchema. + */ + public static FieldSchema getFieldSchemaFromTypeInfo(String fieldName, TypeInfo typeInfo) { + return new FieldSchema( + fieldName, getFieldSchemaTypeFromTypeInfo(typeInfo), "generated by TypeInfoUtils.getFieldSchemaFromTypeInfo" + ); + } + + /** + * The mapping from type name in DDL to the Java class. + */ + public static final Map> TypeNameToClass = new HashMap>(); + static { + TypeNameToClass.put(Constants.BOOLEAN_TYPE_NAME, Boolean.class); + TypeNameToClass.put(Constants.TINYINT_TYPE_NAME, Byte.class); + TypeNameToClass.put(Constants.SMALLINT_TYPE_NAME, Short.class); + TypeNameToClass.put(Constants.INT_TYPE_NAME, Integer.class); + TypeNameToClass.put(Constants.BIGINT_TYPE_NAME, Long.class); + TypeNameToClass.put(Constants.FLOAT_TYPE_NAME, Float.class); + TypeNameToClass.put(Constants.DOUBLE_TYPE_NAME, Double.class); + TypeNameToClass.put(Constants.STRING_TYPE_NAME, String.class); + TypeNameToClass.put(Constants.DATE_TYPE_NAME, java.sql.Date.class); + // These types are not supported yet. + // TypeNameToClass.put(Constants.DATETIME_TYPE_NAME); + // TypeNameToClass.put(Constants.TIMESTAMP_TYPE_NAME); + } + + /** + * Return the primitive type corresponding to the field schema + * @param field The field schema + * @return The TypeInfo object, or null if the field is not a primitive type. + */ + public static TypeInfo getPrimitiveTypeInfoFromFieldSchema(FieldSchema field) { + String type = field.getType(); + + Class c = TypeNameToClass.get(type); + return c == null ? null : TypeInfoFactory.getPrimitiveTypeInfo(c); + } } Index: src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/udf/UDAFAvg.java =================================================================== --- src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/udf/UDAFAvg.java (revision 711472) +++ src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/udf/UDAFAvg.java (working copy) @@ -37,9 +37,9 @@ mCount = 0; } - public boolean aggregate(String o) { - if (o != null && !o.isEmpty()) { - mSum += Double.parseDouble(o); + public boolean aggregate(Double o) { + if (o != null) { + mSum += o; mCount ++; } return true; @@ -60,9 +60,9 @@ return true; } - public String evaluate() { + public Double evaluate() { // This is SQL standard - average of zero items should be null. - return mCount == 0 ? null : String.valueOf(mSum / mCount); + return mCount == 0 ? null : Double.valueOf(mSum / mCount); } } Index: src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/udf/UDAFSum.java =================================================================== --- src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/udf/UDAFSum.java (revision 711472) +++ src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/udf/UDAFSum.java (working copy) @@ -37,30 +37,30 @@ mEmpty = true; } - public boolean aggregate(String o) { - if (o != null && !o.isEmpty()) { - mSum += Double.parseDouble(o); + public boolean aggregate(Double o) { + if (o != null) { + mSum += o; mEmpty = false; } return true; } - public String evaluatePartial() { + public Double evaluatePartial() { // This is SQL standard - sum of zero items should be null. - return mEmpty ? null : new Double(mSum).toString(); + return mEmpty ? null : new Double(mSum); } - public boolean aggregatePartial(String o) { - if (o != null && !o.isEmpty()) { - mSum += Double.parseDouble(o); + public boolean aggregatePartial(Double o) { + if (o != null) { + mSum += o; mEmpty = false; } return true; } - public String evaluate() { + public Double evaluate() { // This is SQL standard - sum of zero items should be null. - return mEmpty ? null : new Double(mSum).toString(); + return mEmpty ? null : new Double(mSum); } } Index: src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToString.java =================================================================== --- src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToString.java (revision 711472) +++ src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToString.java (working copy) @@ -78,12 +78,4 @@ } } - public String evaluate(java.sql.Date i) { - if (i == null) { - return null; - } else { - return i.toString(); - } - } - } Index: src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/udf/UDAFMin.java =================================================================== --- src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/udf/UDAFMin.java (revision 711472) +++ src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/udf/UDAFMin.java (working copy) @@ -37,28 +37,28 @@ mEmpty = true; } - public boolean aggregate(String o) { - if (o != null && !o.isEmpty()) { + public boolean aggregate(Double o) { + if (o != null) { if (mEmpty) { - mMin = Double.parseDouble(o); + mMin = o; mEmpty = false; } else { - mMin = Math.min(mMin, Double.parseDouble(o)); + mMin = Math.min(mMin, o); } } return true; } - public String evaluatePartial() { - return mEmpty ? null : String.valueOf(mMin); + public Double evaluatePartial() { + return mEmpty ? null : Double.valueOf(mMin); } - public boolean aggregatePartial(String o) { + public boolean aggregatePartial(Double o) { return aggregate(o); } - public String evaluate() { - return mEmpty ? null : String.valueOf(mMin); + public Double evaluate() { + return mEmpty ? null : Double.valueOf(mMin); } } Index: src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/udf/UDAFMax.java =================================================================== --- src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/udf/UDAFMax.java (revision 711472) +++ src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/udf/UDAFMax.java (working copy) @@ -37,28 +37,28 @@ mEmpty = true; } - public boolean aggregate(String o) { - if (o != null && !o.isEmpty()) { + public boolean aggregate(Double o) { + if (o != null) { if (mEmpty) { - mMax = Double.parseDouble(o); + mMax = o; mEmpty = false; } else { - mMax = Math.max(mMax, Double.parseDouble(o)); + mMax = Math.max(mMax, o); } } return true; } - public String evaluatePartial() { - return mEmpty ? null : String.valueOf(mMax); + public Double evaluatePartial() { + return mEmpty ? null : Double.valueOf(mMax); } - public boolean aggregatePartial(String o) { + public boolean aggregatePartial(Double o) { return aggregate(o); } - public String evaluate() { - return mEmpty ? null : String.valueOf(mMax); + public Double evaluate() { + return mEmpty ? null : Double.valueOf(mMax); } } Index: src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/udf/UDAFCount.java =================================================================== --- src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/udf/UDAFCount.java (revision 711472) +++ src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/udf/UDAFCount.java (working copy) @@ -42,17 +42,17 @@ return true; } - public String evaluatePartial() { - return Long.valueOf(mCount).toString(); + public Long evaluatePartial() { + return Long.valueOf(mCount); } - public boolean aggregatePartial(String count) { - mCount += Long.parseLong(count); + public boolean aggregatePartial(Long count) { + mCount += count; return true; } - public String evaluate() { - return Long.valueOf(mCount).toString(); + public Long evaluate() { + return Long.valueOf(mCount); } Index: src/contrib/hive/README =================================================================== --- src/contrib/hive/README (revision 711472) +++ src/contrib/hive/README (working copy) @@ -118,6 +118,10 @@ shows the list of columns +hive> DESCRIBE EXTENDED invites; + +shows the list of columns plus any other meta information about the table + Altering tables. Table name can be changed and additional columns can be dropped hive> ALTER TABLE pokes ADD COLUMNS (new_col INT); @@ -258,28 +262,92 @@ STREAMING --------- -hive> FROM invites a INSERT OVERWRITE TABLE events SELECT TRANSFORM(a.foo, a.bar) AS (oof, rab) USING '/bin/cat' WHERE a.ds > '2008-08-09'; +hive> FROM invites a INSERT OVERWRITE TABLE events + > SELECT TRANSFORM(a.foo, a.bar) AS (oof, rab) + > USING '/bin/cat' WHERE a.ds > '2008-08-09'; -This streams the data in the map phase through the script /bin/cat (like hadoop streaming). +This streams the data in the map phase through the script /bin/cat (like hadoop streaming). Similarly - streaming can be used on the reduce side (please see the Hive Tutorial or examples) KNOWN BUGS/ISSUES ----------------- * hive cli may hang for a couple of minutes because of a bug in getting metadata from the derby database. let it run and you'll be fine! -* hive cli does not support multi-line queries. * hive cli creates derby.log in the directory from which it has been invoked. -* DESCRIBE table currently only shows columns in a table. Other metadata like - partitions, buckets etc are not shown. -* LOAD FILE or INSERT INTO TABLE do not validate schemas of the destination tables. * COUNT(*) does not work for now. Use COUNT(1) instead. -* String literals are indicated by single quotes(double quotes are not supported). - So 'is a valid string' while "is not a valid string" in the query language. Hive - does support escaping quotes and semi-colon similar to MySQL. -* Multiple GROUP BYs are not supported in the multi-table table INSERT queries. -* ORDER BY not supported. +* ORDER BY not supported yet. * Only string and thrift types (http://developers.facebook.com/thrift) have been tested. +* When doing Join, please put the table with big number of rows containing the same join key to +the rightmost in the JOIN clause. Otherwise we may see OutOfMemory errors. - FUTURE FEATURES --------------- +* EXPLODE function to generate multiple rows from a column of list type. +* Simpler syntax for running Map/Reduce scripts. +* ORDER BY and SORT BY. +* Table statistics for query optimization. + +Developing Hive using Eclipse +------------------------ +1. Set up hadoop development environment with Eclipse: +http://wiki.apache.org/hadoop/EclipseEnvironment + +2. Download Hive src code from: +http://mirror.facebook.com/facebook/hive + +If hadoop version is 0.17.x or 0.18.x, use +http://mirror.facebook.com/facebook/hive/hadoop-0.17/ + +If hadoop version is 0.19.x or up or trunk, use +http://mirror.facebook.com/facebook/hive/hadoop-0.19/ + +3. Extract the Hive src code to src/contrib/hive, make sure this file (README) + is in src/contrib/hive. + +4. In src/contrib/hive, run "ant package" + +5. In src/contrib/hive, run "ant -logfile test.log test" to make sure + everything works. This test may take 20 minutes. + +6. Add the following list to the Eclipse project's .classpath file: + + + + + + + + + + + + + + + + + + + + + + + + + + + + +7. Develop using Eclipse. + + +Development Tips +------------------------ +* You may change the first line in conf/hive-log4j.properties to the following line to see error messages on the console. +hive.root.logger=INFO,console +Otherwise you will see error messages in /tmp/ +* You may use the following line to test a specific testcase with a specific query file. +ant -Dtestcase=TestParse -Dqfile=udf4.q test +ant -Dtestcase=TestParseNegative -Dqfile=invalid_dot.q test +ant -Dtestcase=TestCliDriver -Dqfile=udf1.q test +ant -Dtestcase=TestNegativeCliDriver -Dqfile=invalid_tbl_name.q test Index: src/contrib/hive/build.xml =================================================================== --- src/contrib/hive/build.xml (revision 711472) +++ src/contrib/hive/build.xml (working copy) @@ -161,10 +161,6 @@ - - - -