diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/UnionOperator.java ql/src/java/org/apache/hadoop/hive/ql/exec/UnionOperator.java index 0f761e8..db10f93 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/UnionOperator.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/UnionOperator.java @@ -92,8 +92,8 @@ protected void initializeOp(Configuration hconf) throws HiveException { columns); for (int c = 0; c < columns; c++) { // can be null for void type - ObjectInspector oi = columnTypeResolvers[c].get(); - outputFieldOIs.add(oi == null ? parentFields[0].get(c).getFieldObjectInspector() : oi); + ObjectInspector fieldOI = parentFields[0].get(c).getFieldObjectInspector(); + outputFieldOIs.add(columnTypeResolvers[c].get(fieldOI)); } // create output row ObjectInspector diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/SimpleFetchOptimizer.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/SimpleFetchOptimizer.java index 906dadf..b5a5a88 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/SimpleFetchOptimizer.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/SimpleFetchOptimizer.java @@ -59,6 +59,7 @@ import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc; +import org.apache.hadoop.hive.ql.plan.ExprNodeNullDesc; import org.apache.hadoop.hive.ql.plan.FetchWork; import org.apache.hadoop.hive.ql.plan.ListSinkDesc; import org.apache.hadoop.hive.ql.plan.OperatorDesc; @@ -244,7 +245,9 @@ private boolean checkExpressions(SelectOperator op) { } private boolean checkExpression(ExprNodeDesc expr) { - if (expr instanceof ExprNodeConstantDesc || expr instanceof ExprNodeColumnDesc) { + if (expr instanceof ExprNodeConstantDesc || + expr instanceof ExprNodeNullDesc|| + expr instanceof ExprNodeColumnDesc) { return true; } diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index 3afc071..12238e8 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -179,7 +179,6 @@ import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeDescUtils; import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc; -import org.apache.hadoop.hive.ql.plan.ExprNodeNullDesc; import org.apache.hadoop.hive.ql.plan.ExtractDesc; import org.apache.hadoop.hive.ql.plan.FileSinkDesc; import org.apache.hadoop.hive.ql.plan.FilterDesc; @@ -3672,11 +3671,6 @@ private static boolean isRegex(String pattern, HiveConf conf) { ArrayList columnNames = new ArrayList(); Map colExprMap = new HashMap(); for (int i = 0; i < col_list.size(); i++) { - // Replace NULL with CAST(NULL AS STRING) - if (col_list.get(i) instanceof ExprNodeNullDesc) { - col_list.set(i, new ExprNodeConstantDesc( - TypeInfoFactory.stringTypeInfo, null)); - } String outputCol = getColumnInternalName(i); colExprMap.put(outputCol, col_list.get(i)); columnNames.add(outputCol); diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFArray.java ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFArray.java index c429367..44eb44c 100644 --- ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFArray.java +++ ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFArray.java @@ -28,7 +28,6 @@ import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters.Converter; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; -import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; /** @@ -59,11 +58,8 @@ public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumen converters = new Converter[arguments.length]; - ObjectInspector returnOI = returnOIResolver.get(); - if (returnOI == null) { - returnOI = PrimitiveObjectInspectorFactory - .getPrimitiveJavaObjectInspector(PrimitiveObjectInspector.PrimitiveCategory.STRING); - } + ObjectInspector returnOI = + returnOIResolver.get(PrimitiveObjectInspectorFactory.javaStringObjectInspector); for (int i = 0; i < arguments.length; i++) { converters[i] = ObjectInspectorConverters.getConverter(arguments[i], returnOI); diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFMap.java ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFMap.java index 0312306..773e1ca 100644 --- ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFMap.java +++ ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFMap.java @@ -82,17 +82,10 @@ public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumen } } - ObjectInspector keyOI = keyOIResolver.get(); - ObjectInspector valueOI = valueOIResolver.get(); - - if (keyOI == null) { - keyOI = PrimitiveObjectInspectorFactory - .getPrimitiveJavaObjectInspector(PrimitiveObjectInspector.PrimitiveCategory.STRING); - } - if (valueOI == null) { - valueOI = PrimitiveObjectInspectorFactory - .getPrimitiveJavaObjectInspector(PrimitiveObjectInspector.PrimitiveCategory.STRING); - } + ObjectInspector keyOI = + keyOIResolver.get(PrimitiveObjectInspectorFactory.javaStringObjectInspector); + ObjectInspector valueOI = + valueOIResolver.get(PrimitiveObjectInspectorFactory.javaStringObjectInspector); converters = new Converter[arguments.length]; diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFSortArray.java ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFSortArray.java index a43867c..40d1bdd 100644 --- ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFSortArray.java +++ ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFSortArray.java @@ -82,10 +82,7 @@ public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumen ((ListObjectInspector)(arguments[0])).getListElementObjectInspector(); argumentOIs = arguments; converters = new Converter[arguments.length]; - ObjectInspector returnOI = returnOIResolver.get(); - if (returnOI == null) { - returnOI = elementObjectInspector; - } + ObjectInspector returnOI = returnOIResolver.get(elementObjectInspector); converters[0] = ObjectInspectorConverters.getConverter(elementObjectInspector, returnOI); return ObjectInspectorFactory.getStandardListObjectInspector(returnOI); diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFUtils.java ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFUtils.java index 833452d..f197afa 100644 --- ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFUtils.java +++ ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFUtils.java @@ -22,7 +22,6 @@ import java.lang.reflect.Method; import java.lang.reflect.ParameterizedType; import java.lang.reflect.Type; -import java.nio.ByteBuffer; import java.util.HashMap; import org.apache.hadoop.hive.ql.exec.FunctionRegistry; @@ -41,6 +40,7 @@ import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils.ObjectInspectorCopyOption; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; import org.apache.hadoop.hive.serde2.objectinspector.primitive.VoidObjectInspector; import org.apache.hadoop.hive.serde2.typeinfo.BaseCharTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo; @@ -191,7 +191,11 @@ private boolean update(ObjectInspector oi, boolean isUnionAll) throws UDFArgumen * Returns the ObjectInspector of the return value. */ public ObjectInspector get() { - return returnObjectInspector; + return get(PrimitiveObjectInspectorFactory.javaVoidObjectInspector); + } + + public ObjectInspector get(ObjectInspector defaultOI) { + return returnObjectInspector != null ? returnObjectInspector : defaultOI; } /** diff --git ql/src/test/queries/clientpositive/udtf_stack.q ql/src/test/queries/clientpositive/udtf_stack.q index e7cbae0..1643ab5 100644 --- ql/src/test/queries/clientpositive/udtf_stack.q +++ ql/src/test/queries/clientpositive/udtf_stack.q @@ -5,3 +5,7 @@ EXPLAIN SELECT x, y FROM src LATERAL VIEW STACK(2, 'x', array(1), 'z', array(4)) SELECT x, y FROM src LATERAL VIEW STACK(2, 'x', array(1), 'z') a AS x, y LIMIT 2; SELECT x, y FROM src LATERAL VIEW STACK(2, 'x', array(1), 'z', array(4)) a AS x, y LIMIT 2; + +EXPLAIN +SELECT stack(1, "en", "dbpedia", NULL ); +SELECT stack(1, "en", "dbpedia", NULL ); \ No newline at end of file diff --git ql/src/test/results/clientpositive/annotate_stats_select.q.out ql/src/test/results/clientpositive/annotate_stats_select.q.out index 8ef4964..8fbb208 100644 --- ql/src/test/results/clientpositive/annotate_stats_select.q.out +++ ql/src/test/results/clientpositive/annotate_stats_select.q.out @@ -282,7 +282,7 @@ STAGE PLANS: alias: alltypes_orc Statistics: Num rows: 2 Data size: 1686 Basic stats: COMPLETE Column stats: COMPLETE Select Operator - expressions: null (type: string) + expressions: null (type: void) outputColumnNames: _col0 Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE ListSink @@ -497,33 +497,21 @@ POSTHOOK: query: -- numRows: 2 rawDataSize: 112 explain select cast("1970-12-31 15:59:58.174" as DATE) from alltypes_orc POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 + Stage-0 is a root stage STAGE PLANS: - Stage: Stage-1 - Map Reduce - Map Operator Tree: - TableScan - alias: alltypes_orc - Statistics: Num rows: 2 Data size: 1686 Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - expressions: null (type: void) - outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - Stage: Stage-0 Fetch Operator limit: -1 Processor Tree: - ListSink + TableScan + alias: alltypes_orc + Statistics: Num rows: 2 Data size: 1686 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: null (type: void) + outputColumnNames: _col0 + Statistics: Num rows: 2 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + ListSink PREHOOK: query: -- numRows: 2 rawDataSize: 224 explain select cast("58.174" as DECIMAL) from alltypes_orc diff --git ql/src/test/results/clientpositive/input9.q.out ql/src/test/results/clientpositive/input9.q.out index ec1bcab..4666787 100644 --- ql/src/test/results/clientpositive/input9.q.out +++ ql/src/test/results/clientpositive/input9.q.out @@ -35,7 +35,7 @@ STAGE PLANS: predicate: (null = null) (type: boolean) Statistics: Num rows: 12 Data size: 91 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: null (type: string), UDFToInteger(key) (type: int) + expressions: null (type: void), UDFToInteger(key) (type: int) outputColumnNames: _col0, _col1 Statistics: Num rows: 12 Data size: 91 Basic stats: COMPLETE Column stats: NONE File Output Operator diff --git ql/src/test/results/clientpositive/udtf_stack.q.out ql/src/test/results/clientpositive/udtf_stack.q.out index 43f0a76..4e9c2fd 100644 --- ql/src/test/results/clientpositive/udtf_stack.q.out +++ ql/src/test/results/clientpositive/udtf_stack.q.out @@ -157,3 +157,51 @@ POSTHOOK: Input: default@src #### A masked pattern was here #### x [1] z [4] +PREHOOK: query: EXPLAIN +SELECT stack(1, "en", "dbpedia", NULL ) +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN +SELECT stack(1, "en", "dbpedia", NULL ) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: _dummy_table + Row Limit Per Split: 1 + Statistics: Num rows: 0 Data size: 1 Basic stats: PARTIAL Column stats: COMPLETE + Select Operator + expressions: 1 (type: int), 'en' (type: string), 'dbpedia' (type: string), null (type: void) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 0 Data size: 1 Basic stats: PARTIAL Column stats: COMPLETE + UDTF Operator + Statistics: Num rows: 0 Data size: 1 Basic stats: PARTIAL Column stats: COMPLETE + function name: stack + File Output Operator + compressed: false + Statistics: Num rows: 0 Data size: 1 Basic stats: PARTIAL Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT stack(1, "en", "dbpedia", NULL ) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +#### A masked pattern was here #### +POSTHOOK: query: SELECT stack(1, "en", "dbpedia", NULL ) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +#### A masked pattern was here #### +en dbpedia NULL