diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagateProcFactory.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagateProcFactory.java index 410735c..f02f3a7 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagateProcFactory.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagateProcFactory.java @@ -76,8 +76,10 @@ import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPOr; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFWhen; import org.apache.hadoop.hive.serde.serdeConstants; +import org.apache.hadoop.hive.serde2.objectinspector.ConstantObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters; +import org.apache.hadoop.hive.serde2.objectinspector.StandardConstantStructObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters.Converter; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; @@ -761,11 +763,16 @@ private static ExprNodeDesc evaluateFunction(GenericUDF udf, List typeInfo.getTypeName().contains(serdeConstants.CHAR_TYPE_NAME)) { return new ExprNodeConstantDesc(typeInfo, o); } - } else if (PrimitiveObjectInspectorUtils.isPrimitiveJavaClass(clz)) { - + } else if (oi instanceof StandardConstantStructObjectInspector) { + ConstantObjectInspector coi = (ConstantObjectInspector) oi; + TypeInfo structType = TypeInfoUtils.getTypeInfoFromObjectInspector(coi); + return new ExprNodeConstantDesc(structType, o); + } else if (!PrimitiveObjectInspectorUtils.isPrimitiveJavaClass(clz)) { + // fall through below } else { if (LOG.isErrorEnabled()) { - LOG.error("Unable to evaluate " + udf + ". Return value unrecoginizable."); + LOG.error("Unable to evaluate " + udf + + ". Return value unrecoginizable."); } return null; } diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeConstantDesc.java ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeConstantDesc.java index 2674fe3..a5221a2 100755 --- ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeConstantDesc.java +++ ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeConstantDesc.java @@ -19,12 +19,15 @@ package org.apache.hadoop.hive.ql.plan; import java.io.Serializable; +import java.util.List; import org.apache.commons.lang.builder.HashCodeBuilder; import org.apache.hadoop.hive.serde.serdeConstants; import org.apache.hadoop.hive.serde2.objectinspector.ConstantObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; import org.apache.hadoop.hive.serde2.typeinfo.BaseCharTypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; @@ -73,6 +76,7 @@ public ExprNodeConstantDesc(Object value) { } public void setValue(Object value) { + // Kryo setter this.value = value; } @@ -92,8 +96,7 @@ public String toString() { return "Const " + typeInfo.toString() + " " + value; } - @Override - public String getExprString() { + private static String getFormatted(TypeInfo typeInfo, Object value) { if (value == null) { return "null"; } @@ -109,8 +112,28 @@ public String getExprString() { hexChars[j * 2 + 1] = hexArray[v & 0x0F]; } return new String(hexChars); + } + return value.toString(); + } + + @Override + public String getExprString() { + if (typeInfo.getCategory() == Category.PRIMITIVE) { + return getFormatted(typeInfo, value); + } else if (typeInfo.getCategory() == Category.STRUCT) { + StringBuilder sb = new StringBuilder(); + sb.append("const struct("); + List items = (List) getWritableObjectInspector().getWritableConstantValue(); + List structTypes = ((StructTypeInfo) typeInfo).getAllStructFieldTypeInfos(); + for (int i = 0; i < structTypes.size(); i++) { + final Object o = (i < items.size()) ? items.get(i) : null; + sb.append(getFormatted(structTypes.get(i), o)).append(","); + } + sb.setCharAt(sb.length() - 1, ')'); + return sb.toString(); } else { - return value.toString(); + // unknown type + return toString(); } } diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFIn.java ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFIn.java index 56ac3e1..7660ca4 100644 --- ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFIn.java +++ ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFIn.java @@ -60,7 +60,8 @@ public class GenericUDFIn extends GenericUDF { private transient ObjectInspector[] argumentOIs; - private Set constantInSet; + // this set is a copy of the arguments objects - avoid serializing + private transient Set constantInSet; private boolean isInSetConstant = true; //are variables from IN(...) constant private final BooleanWritable bw = new BooleanWritable(); diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFStruct.java ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFStruct.java index 7df3f7d..4654512 100644 --- ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFStruct.java +++ ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFStruct.java @@ -24,9 +24,9 @@ import org.apache.hadoop.hive.ql.exec.UDFArgumentException; import org.apache.hadoop.hive.ql.exec.Description; import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.serde2.objectinspector.ConstantObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; -import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; @Description(name = "struct", value = "_FUNC_(col1, col2, col3, ...) - Creates a struct with the given field values") @@ -44,9 +44,21 @@ public ObjectInspector initialize(ObjectInspector[] arguments) for (int f = 1; f <= numFields; f++) { fname.add("col" + f); } - StructObjectInspector soi = - ObjectInspectorFactory.getStandardStructObjectInspector(fname, Arrays.asList(arguments)); - return soi; + boolean constantStruct = true; + for (int i = 0; i < arguments.length; i++) { + ObjectInspector oi = arguments[i]; + constantStruct &= (oi instanceof ConstantObjectInspector); + if (constantStruct) { + ret[i] = ((ConstantObjectInspector) oi).getWritableConstantValue(); + } + } + if (constantStruct) { + return ObjectInspectorFactory.getStandardConstantStructObjectInspector(fname, + Arrays.asList(arguments), Arrays.asList(ret)); + } else { + return ObjectInspectorFactory.getStandardStructObjectInspector(fname, + Arrays.asList(arguments)); + } } @Override diff --git serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorUtils.java serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorUtils.java index 64dd512..b88a0fa 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorUtils.java +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorUtils.java @@ -1084,6 +1084,9 @@ public static ConstantObjectInspector getConstantObjectInspector(ObjectInspector fieldObjectInspectors.add(getStandardObjectInspector(f .getFieldObjectInspector(), ObjectInspectorCopyOption.WRITABLE)); } + if (writableValue.getClass().isArray()) { + writableValue = java.util.Arrays.asList((Object[])writableValue); + } return ObjectInspectorFactory.getStandardConstantStructObjectInspector( fieldNames, fieldObjectInspectors,