diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties index 7a70c9c..70b2335 100644 --- a/itests/src/test/resources/testconfiguration.properties +++ b/itests/src/test/resources/testconfiguration.properties @@ -501,6 +501,7 @@ minillaplocal.query.files=acid_globallimit.q,\ groupby2.q,\ hybridgrace_hashjoin_1.q,\ hybridgrace_hashjoin_2.q,\ + is_distinct_from.q,\ infer_bucket_sort_bucketed_table.q,\ input16_cc.q,\ insert_dir_distcp.q,\ diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java index ccfb455..24fb94e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java @@ -357,6 +357,8 @@ system.registerGenericUDF("isnull", GenericUDFOPNull.class); system.registerGenericUDF("isnotnull", GenericUDFOPNotNull.class); + system.registerGenericUDF("TOK_ISDISTINCTFROM", GenericUDFOPIsDistinctFrom.class); + system.registerGenericUDF("TOK_ISNOTDISTINCTFROM", GenericUDFOPIsNotDistinctFrom.class); system.registerGenericUDF("if", GenericUDFIf.class); system.registerGenericUDF("in", GenericUDFIn.class); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/SqlFunctionConverter.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/SqlFunctionConverter.java index 85450c9..be49755 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/SqlFunctionConverter.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/SqlFunctionConverter.java @@ -352,6 +352,8 @@ private static String getName(GenericUDF hiveUDF) { registerFunction("struct", SqlStdOperatorTable.ROW, hToken(HiveParser.Identifier, "struct")); registerFunction("isnotnull", SqlStdOperatorTable.IS_NOT_NULL, hToken(HiveParser.TOK_ISNOTNULL, "TOK_ISNOTNULL")); registerFunction("isnull", SqlStdOperatorTable.IS_NULL, hToken(HiveParser.TOK_ISNULL, "TOK_ISNULL")); + registerFunction("TOK_ISDISTINCTFROM", SqlStdOperatorTable.IS_DISTINCT_FROM, hToken(HiveParser.TOK_ISDISTINCTFROM, "TOK_ISNULL")); + registerFunction("TOK_ISNOTDISTINCTFROM", SqlStdOperatorTable.IS_NOT_DISTINCT_FROM, hToken(HiveParser.TOK_ISNOTDISTINCTFROM, "TOK_ISNULL")); registerFunction("when", SqlStdOperatorTable.CASE, hToken(HiveParser.Identifier, "when")); registerDuplicateFunction("case", SqlStdOperatorTable.CASE, hToken(HiveParser.Identifier, "when")); // timebased diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g b/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g index d98a663..108b714 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g @@ -397,6 +397,8 @@ TOK_OPERATOR; TOK_EXPRESSION; TOK_DETAIL; TOK_BLOCKING; +TOK_ISDISTINCTFROM; +TOK_ISNOTDISTINCTFROM; } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g b/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g index 8c4ee8a..64e9120 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g @@ -585,9 +585,16 @@ precedenceSimilarExpressionPartNot[CommonTree t] precedenceSimilarExpressionAtom[$t] ; +isDistinctFrom + : + KW_IS (a=KW_NOT)? KW_DISTINCT KW_FROM + -> {$a !=null}? ^(TOK_ISNOTDISTINCTFROM) + -> ^(TOK_ISDISTINCTFROM) + ; + precedenceEqualOperator : - EQUAL | EQUAL_NS | NOTEQUAL + EQUAL | EQUAL_NS | NOTEQUAL | isDistinctFrom ; precedenceEqualExpression diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPIsDistinctFrom.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPIsDistinctFrom.java new file mode 100644 index 0000000..78ef761 --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPIsDistinctFrom.java @@ -0,0 +1,48 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.udf.generic; + +import org.apache.hadoop.hive.ql.exec.Description; +import org.apache.hadoop.hive.ql.metadata.HiveException; + +@Description(name = "is distinct from", value = "a _FUNC_ b - Returns same result with NOT EQUAL(<>) operator " + + "for non-null operands, but returns FALSE if both are NULL, TRUE if one of the them is NULL") +@NDV(maxNdv = 2) +public class GenericUDFOPIsDistinctFrom extends GenericUDFOPNotEqual{ + + @Override + public Object evaluate(DeferredObject[] arguments) throws HiveException { + Object o0 = arguments[0].get(); + Object o1 = arguments[1].get(); + if (o0 == null && o1 == null) { + result.set(false); + return result; + } + if (o0 == null || o1 == null) { + result.set(true); + return result; + } + return super.evaluate(arguments); + } + + @Override + public GenericUDF negative() { + return new GenericUDFOPIsNotDistinctFrom(); + } +} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPIsNotDistinctFrom.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPIsNotDistinctFrom.java new file mode 100644 index 0000000..3b666da --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPIsNotDistinctFrom.java @@ -0,0 +1,45 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.udf.generic; + +import org.apache.hadoop.hive.ql.metadata.HiveException; + +// this function is for internal use only +public class GenericUDFOPIsNotDistinctFrom extends GenericUDFOPEqual { + + @Override + public Object evaluate(DeferredObject[] arguments) throws HiveException { + Object o0 = arguments[0].get(); + Object o1 = arguments[1].get(); + if (o0 == null && o1 == null) { + result.set(true); + return result; + } + if (o0 == null || o1 == null) { + result.set(false); + return result; + } + return super.evaluate(arguments); + } + + @Override + public GenericUDF negative() { + return new GenericUDFOPIsDistinctFrom(); + } +} diff --git a/ql/src/test/queries/clientpositive/is_distinct_from.q b/ql/src/test/queries/clientpositive/is_distinct_from.q new file mode 100644 index 0000000..e62f48b --- /dev/null +++ b/ql/src/test/queries/clientpositive/is_distinct_from.q @@ -0,0 +1,23 @@ +explain select 1 is distinct from 1, + 1 is distinct from 2, + 1 is distinct from null, + null is distinct from null + from part; + +select 1 is distinct from 1, + 1 is distinct from 2, + 1 is distinct from null, + null is distinct from null + from part; + +explain select 1 is not distinct from 1, + 1 is not distinct from 2, + 1 is not distinct from null, + null is not distinct from null + from part; + +select 1 is not distinct from 1, + 1 is not distinct from 2, + 1 is not distinct from null, + null is not distinct from null + from part; diff --git a/ql/src/test/results/clientpositive/llap/is_distinct_from.q.out b/ql/src/test/results/clientpositive/llap/is_distinct_from.q.out new file mode 100644 index 0000000..14cc130 --- /dev/null +++ b/ql/src/test/results/clientpositive/llap/is_distinct_from.q.out @@ -0,0 +1,138 @@ +PREHOOK: query: explain select 1 is distinct from 1, + 1 is distinct from 2, + 1 is distinct from null, + null is distinct from null + from part +PREHOOK: type: QUERY +POSTHOOK: query: explain select 1 is distinct from 1, + 1 is distinct from 2, + 1 is distinct from null, + null is distinct from null + from part +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: part + Select Operator + expressions: false (type: boolean), true (type: boolean), true (type: boolean), false (type: boolean) + outputColumnNames: _col0, _col1, _col2, _col3 + ListSink + +PREHOOK: query: select 1 is distinct from 1, + 1 is distinct from 2, + 1 is distinct from null, + null is distinct from null + from part +PREHOOK: type: QUERY +PREHOOK: Input: default@part +#### A masked pattern was here #### +POSTHOOK: query: select 1 is distinct from 1, + 1 is distinct from 2, + 1 is distinct from null, + null is distinct from null + from part +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part +#### A masked pattern was here #### +false true true false +false true true false +false true true false +false true true false +false true true false +false true true false +false true true false +false true true false +false true true false +false true true false +false true true false +false true true false +false true true false +false true true false +false true true false +false true true false +false true true false +false true true false +false true true false +false true true false +false true true false +false true true false +false true true false +false true true false +false true true false +false true true false +PREHOOK: query: explain select 1 is not distinct from 1, + 1 is not distinct from 2, + 1 is not distinct from null, + null is not distinct from null + from part +PREHOOK: type: QUERY +POSTHOOK: query: explain select 1 is not distinct from 1, + 1 is not distinct from 2, + 1 is not distinct from null, + null is not distinct from null + from part +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: part + Select Operator + expressions: true (type: boolean), false (type: boolean), false (type: boolean), true (type: boolean) + outputColumnNames: _col0, _col1, _col2, _col3 + ListSink + +PREHOOK: query: select 1 is not distinct from 1, + 1 is not distinct from 2, + 1 is not distinct from null, + null is not distinct from null + from part +PREHOOK: type: QUERY +PREHOOK: Input: default@part +#### A masked pattern was here #### +POSTHOOK: query: select 1 is not distinct from 1, + 1 is not distinct from 2, + 1 is not distinct from null, + null is not distinct from null + from part +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part +#### A masked pattern was here #### +true false false true +true false false true +true false false true +true false false true +true false false true +true false false true +true false false true +true false false true +true false false true +true false false true +true false false true +true false false true +true false false true +true false false true +true false false true +true false false true +true false false true +true false false true +true false false true +true false false true +true false false true +true false false true +true false false true +true false false true +true false false true +true false false true