Index: ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeEvaluator.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeEvaluator.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeEvaluator.java @@ -40,4 +40,11 @@ */ public abstract Object evaluate(Object row) throws HiveException; + /** + * Return whether this node (and all children nodes) are deterministic. + */ + public boolean isDeterministic() { + return true; + } + } Index: ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeGenericFuncEvaluator.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeGenericFuncEvaluator.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeGenericFuncEvaluator.java @@ -138,14 +138,24 @@ } @Override + public boolean isDeterministic() { + boolean result = FunctionRegistry.isDeterministic(genericUDF); + for (ExprNodeEvaluator child : children) { + result = result && child.isDeterministic(); + } + return result; + } + + @Override public Object evaluate(Object row) throws HiveException { rowObject = row; - if (ObjectInspectorUtils.isConstantObjectInspector(outputOI)) { + if (ObjectInspectorUtils.isConstantObjectInspector(outputOI) && + isDeterministic()) { // The output of this UDF is constant, so don't even bother evaluating. return ((ConstantObjectInspector)outputOI).getWritableConstantValue(); } if (isEager) { for (int i = 0; i < deferredChildren.length; i++) { ((EagerExprObject) deferredChildren[i]).evaluate(); } } Index: ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java @@ -146,11 +146,12 @@ import org.apache.hadoop.hive.ql.udf.generic.GenericUDF; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFArray; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFArrayContains; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDFAssertTrue; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFBridge; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFCase; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFCoalesce; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFConcatWS; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFEWAHBitmapAnd; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFEWAHBitmapEmpty; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFEWAHBitmapOr; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFElt; @@ -414,11 +415,12 @@ registerGenericUDF("reflect", GenericUDFReflect.class); registerGenericUDF("array", GenericUDFArray.class); + registerGenericUDF("assert_true", GenericUDFAssertTrue.class); registerGenericUDF("map", GenericUDFMap.class); registerGenericUDF("struct", GenericUDFStruct.class); registerGenericUDF("named_struct", GenericUDFNamedStruct.class); registerGenericUDF("create_union", GenericUDFUnion.class); registerGenericUDF("case", GenericUDFCase.class); registerGenericUDF("when", GenericUDFWhen.class); registerGenericUDF("hash", GenericUDFHash.class); Index: ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFAssertTrue.java =================================================================== --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFAssertTrue.java @@ -0,0 +1,89 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.udf.generic; + +import org.apache.hadoop.hive.ql.exec.Description; +import org.apache.hadoop.hive.ql.exec.UDFArgumentException; +import org.apache.hadoop.hive.ql.exec.UDFArgumentLengthException; +import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.udf.UDFType; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDF; + +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; + +import org.apache.hadoop.io.BooleanWritable; + +/** + * GenericUDFAssertTrue + */ +@Description(name = "assert_true", + value = "_FUNC_(condition) - " + + "Throw an exception if 'condition' is not true.", + extended = "Example:\n " + + " > SELECT _FUNC_(x >= 0) FROM src LIMIT 1;\n" + + " NULL") +@UDFType(deterministic = false) +public class GenericUDFAssertTrue extends GenericUDF { + private ObjectInspectorConverters.Converter conditionConverter = null; + + @Override + public ObjectInspector initialize(ObjectInspector[] arguments) + throws UDFArgumentException { + if (arguments.length != 1) { + throw new UDFArgumentLengthException( + "ASSERT_TRUE() expects one argument."); + } + if (arguments[0].getCategory() != Category.PRIMITIVE) { + throw new UDFArgumentTypeException(0, + "Argument to ASSERT_TRUE() should be primitive."); + } + conditionConverter = ObjectInspectorConverters.getConverter(arguments[0], + PrimitiveObjectInspectorFactory.writableBooleanObjectInspector); + + return PrimitiveObjectInspectorFactory.writableVoidObjectInspector; + } + + @Override + public Object evaluate(DeferredObject[] arguments) throws HiveException { + BooleanWritable condition = + (BooleanWritable)conditionConverter.convert(arguments[0].get()); + if (condition == null || !condition.get()) { + throw new HiveException("ASSERT_TRUE(): assertion failed."); + } + return null; + } + + @Override + public String getDisplayString(String[] children) { + StringBuilder sb = new StringBuilder(); + sb.append("assert_true("); + for (int i = 0; i < children.length; ++i) { + sb.append(children[i]); + if (i != children.length - 1) { + sb.append(", "); + } + } + sb.append(")"); + return sb.toString(); + } +} Index: ql/src/test/queries/clientnegative/udf_assert_true.q =================================================================== --- /dev/null +++ ql/src/test/queries/clientnegative/udf_assert_true.q @@ -0,0 +1,7 @@ +DESCRIBE FUNCTION ASSERT_TRUE; + +EXPLAIN SELECT ASSERT_TRUE(x > 0) FROM src LATERAL VIEW EXPLODE(ARRAY(1, 2)) a AS x LIMIT 2; +SELECT ASSERT_TRUE(x > 0) FROM src LATERAL VIEW EXPLODE(ARRAY(1, 2)) a AS x LIMIT 2; + +EXPLAIN SELECT ASSERT_TRUE(x < 2) FROM src LATERAL VIEW EXPLODE(ARRAY(1, 2)) a AS x LIMIT 2; +SELECT ASSERT_TRUE(x < 2) FROM src LATERAL VIEW EXPLODE(ARRAY(1, 2)) a AS x LIMIT 2; Index: ql/src/test/queries/clientnegative/udf_assert_true2.q =================================================================== --- /dev/null +++ ql/src/test/queries/clientnegative/udf_assert_true2.q @@ -0,0 +1,2 @@ +EXPLAIN SELECT 1 + ASSERT_TRUE(x < 2) FROM src LATERAL VIEW EXPLODE(ARRAY(1, 2)) a AS x LIMIT 2; +SELECT 1 + ASSERT_TRUE(x < 2) FROM src LATERAL VIEW EXPLODE(ARRAY(1, 2)) a AS x LIMIT 2; Index: ql/src/test/results/clientnegative/udf_assert_true.q.out =================================================================== --- /dev/null +++ ql/src/test/results/clientnegative/udf_assert_true.q.out @@ -0,0 +1,154 @@ +PREHOOK: query: DESCRIBE FUNCTION ASSERT_TRUE +PREHOOK: type: DESCFUNCTION +POSTHOOK: query: DESCRIBE FUNCTION ASSERT_TRUE +POSTHOOK: type: DESCFUNCTION +ASSERT_TRUE(condition) - Throw an exception if 'condition' is not true. +PREHOOK: query: EXPLAIN SELECT ASSERT_TRUE(x > 0) FROM src LATERAL VIEW EXPLODE(ARRAY(1, 2)) a AS x LIMIT 2 +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN SELECT ASSERT_TRUE(x > 0) FROM src LATERAL VIEW EXPLODE(ARRAY(1, 2)) a AS x LIMIT 2 +POSTHOOK: type: QUERY +ABSTRACT SYNTAX TREE: + (TOK_QUERY (TOK_FROM (TOK_LATERAL_VIEW (TOK_SELECT (TOK_SELEXPR (TOK_FUNCTION EXPLODE (TOK_FUNCTION ARRAY 1 2)) x (TOK_TABALIAS a))) (TOK_TABREF (TOK_TABNAME src)))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_FUNCTION ASSERT_TRUE (> (TOK_TABLE_OR_COL x) 0)))) (TOK_LIMIT 2))) + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Alias -> Map Operator Tree: + src + TableScan + alias: src + Lateral View Forward + Select Operator + SELECT * : (no compute) + Lateral View Join Operator + outputColumnNames: _col0, _col1, _col2 + Select Operator + expressions: + expr: assert_true((_col2 > 0)) + type: void + outputColumnNames: _col0 + Limit + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + Select Operator + expressions: + expr: array(1,2) + type: array + outputColumnNames: _col0 + UDTF Operator + function name: explode + Lateral View Join Operator + outputColumnNames: _col0, _col1, _col2 + Select Operator + expressions: + expr: assert_true((_col2 > 0)) + type: void + outputColumnNames: _col0 + Limit + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + + Stage: Stage-0 + Fetch Operator + limit: 2 + + +PREHOOK: query: SELECT ASSERT_TRUE(x > 0) FROM src LATERAL VIEW EXPLODE(ARRAY(1, 2)) a AS x LIMIT 2 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: file:/var/folders/71/h_j6fpg10r33hvx1lcxlgttcw61_4s/T/jonchang/hive_2011-11-05_15-55-33_934_5378238297627471477/-mr-10000 +POSTHOOK: query: SELECT ASSERT_TRUE(x > 0) FROM src LATERAL VIEW EXPLODE(ARRAY(1, 2)) a AS x LIMIT 2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: file:/var/folders/71/h_j6fpg10r33hvx1lcxlgttcw61_4s/T/jonchang/hive_2011-11-05_15-55-33_934_5378238297627471477/-mr-10000 +NULL +NULL +PREHOOK: query: EXPLAIN SELECT ASSERT_TRUE(x < 2) FROM src LATERAL VIEW EXPLODE(ARRAY(1, 2)) a AS x LIMIT 2 +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN SELECT ASSERT_TRUE(x < 2) FROM src LATERAL VIEW EXPLODE(ARRAY(1, 2)) a AS x LIMIT 2 +POSTHOOK: type: QUERY +ABSTRACT SYNTAX TREE: + (TOK_QUERY (TOK_FROM (TOK_LATERAL_VIEW (TOK_SELECT (TOK_SELEXPR (TOK_FUNCTION EXPLODE (TOK_FUNCTION ARRAY 1 2)) x (TOK_TABALIAS a))) (TOK_TABREF (TOK_TABNAME src)))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_FUNCTION ASSERT_TRUE (< (TOK_TABLE_OR_COL x) 2)))) (TOK_LIMIT 2))) + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Alias -> Map Operator Tree: + src + TableScan + alias: src + Lateral View Forward + Select Operator + SELECT * : (no compute) + Lateral View Join Operator + outputColumnNames: _col0, _col1, _col2 + Select Operator + expressions: + expr: assert_true((_col2 < 2)) + type: void + outputColumnNames: _col0 + Limit + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + Select Operator + expressions: + expr: array(1,2) + type: array + outputColumnNames: _col0 + UDTF Operator + function name: explode + Lateral View Join Operator + outputColumnNames: _col0, _col1, _col2 + Select Operator + expressions: + expr: assert_true((_col2 < 2)) + type: void + outputColumnNames: _col0 + Limit + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + + Stage: Stage-0 + Fetch Operator + limit: 2 + + +PREHOOK: query: SELECT ASSERT_TRUE(x < 2) FROM src LATERAL VIEW EXPLODE(ARRAY(1, 2)) a AS x LIMIT 2 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: file:/var/folders/71/h_j6fpg10r33hvx1lcxlgttcw61_4s/T/jonchang/hive_2011-11-05_15-55-39_927_2246328313093384798/-mr-10000 +Execution failed with exit status: 2 +Obtaining error information + +Task failed! +Task ID: + Stage-1 + +Logs: + +/Users/jonchang/hive/build/ql/tmp//hive.log +FAILED: Execution Error, return code 2 from org.apache.hadoop.hive.ql.exec.MapRedTask Index: ql/src/test/results/clientnegative/udf_assert_true2.q.out =================================================================== --- /dev/null +++ ql/src/test/results/clientnegative/udf_assert_true2.q.out @@ -0,0 +1,77 @@ +PREHOOK: query: EXPLAIN SELECT 1 + ASSERT_TRUE(x < 2) FROM src LATERAL VIEW EXPLODE(ARRAY(1, 2)) a AS x LIMIT 2 +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN SELECT 1 + ASSERT_TRUE(x < 2) FROM src LATERAL VIEW EXPLODE(ARRAY(1, 2)) a AS x LIMIT 2 +POSTHOOK: type: QUERY +ABSTRACT SYNTAX TREE: + (TOK_QUERY (TOK_FROM (TOK_LATERAL_VIEW (TOK_SELECT (TOK_SELEXPR (TOK_FUNCTION EXPLODE (TOK_FUNCTION ARRAY 1 2)) x (TOK_TABALIAS a))) (TOK_TABREF (TOK_TABNAME src)))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (+ 1 (TOK_FUNCTION ASSERT_TRUE (< (TOK_TABLE_OR_COL x) 2))))) (TOK_LIMIT 2))) + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Alias -> Map Operator Tree: + src + TableScan + alias: src + Lateral View Forward + Select Operator + SELECT * : (no compute) + Lateral View Join Operator + outputColumnNames: _col0, _col1, _col2 + Select Operator + expressions: + expr: (1 + assert_true((_col2 < 2))) + type: int + outputColumnNames: _col0 + Limit + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + Select Operator + expressions: + expr: array(1,2) + type: array + outputColumnNames: _col0 + UDTF Operator + function name: explode + Lateral View Join Operator + outputColumnNames: _col0, _col1, _col2 + Select Operator + expressions: + expr: (1 + assert_true((_col2 < 2))) + type: int + outputColumnNames: _col0 + Limit + File Output Operator + compressed: false + GlobalTableId: 0 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + + Stage: Stage-0 + Fetch Operator + limit: 2 + + +PREHOOK: query: SELECT 1 + ASSERT_TRUE(x < 2) FROM src LATERAL VIEW EXPLODE(ARRAY(1, 2)) a AS x LIMIT 2 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: file:/var/folders/71/h_j6fpg10r33hvx1lcxlgttcw61_4s/T/jonchang/hive_2011-11-09_22-43-58_159_7985036840632043879/-mr-10000 +Execution failed with exit status: 2 +Obtaining error information + +Task failed! +Task ID: + Stage-1 + +Logs: + +/Users/jonchang/hive/build/ql/tmp//hive.log +FAILED: Execution Error, return code 2 from org.apache.hadoop.hive.ql.exec.MapRedTask Index: ql/src/test/results/clientpositive/show_functions.q.out =================================================================== --- ql/src/test/results/clientpositive/show_functions.q.out +++ ql/src/test/results/clientpositive/show_functions.q.out @@ -25,11 +25,12 @@ array_contains ascii asin +assert_true atan avg bigint bin binary boolean case ceil @@ -195,11 +196,12 @@ PREHOOK: type: SHOWFUNCTIONS POSTHOOK: query: SHOW FUNCTIONS '.*e$' POSTHOOK: type: SHOWFUNCTIONS +assert_true case coalesce double e explode from_unixtime json_tuple lcase