diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties index 7a70c9c..70b2335 100644 --- a/itests/src/test/resources/testconfiguration.properties +++ b/itests/src/test/resources/testconfiguration.properties @@ -501,6 +501,7 @@ minillaplocal.query.files=acid_globallimit.q,\ groupby2.q,\ hybridgrace_hashjoin_1.q,\ hybridgrace_hashjoin_2.q,\ + is_distinct_from.q,\ infer_bucket_sort_bucketed_table.q,\ input16_cc.q,\ insert_dir_distcp.q,\ diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java index ccfb455..01c02cb 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java @@ -357,6 +357,8 @@ system.registerGenericUDF("isnull", GenericUDFOPNull.class); system.registerGenericUDF("isnotnull", GenericUDFOPNotNull.class); + system.registerGenericUDF("is distinct from", GenericUDFOPNotEqualNS.class); + system.registerGenericUDF("is not distinct from", GenericUDFOPEqualNS.class); system.registerGenericUDF("if", GenericUDFIf.class); system.registerGenericUDF("in", GenericUDFIn.class); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/SqlFunctionConverter.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/SqlFunctionConverter.java index 85450c9..2b0d139 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/SqlFunctionConverter.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/SqlFunctionConverter.java @@ -352,6 +352,8 @@ private static String getName(GenericUDF hiveUDF) { registerFunction("struct", SqlStdOperatorTable.ROW, hToken(HiveParser.Identifier, "struct")); registerFunction("isnotnull", SqlStdOperatorTable.IS_NOT_NULL, hToken(HiveParser.TOK_ISNOTNULL, "TOK_ISNOTNULL")); registerFunction("isnull", SqlStdOperatorTable.IS_NULL, hToken(HiveParser.TOK_ISNULL, "TOK_ISNULL")); + registerFunction("is distinct from", SqlStdOperatorTable.IS_DISTINCT_FROM, hToken(HiveParser.TOK_ISDISTINCTFROM, "TOK_ISNULL")); + registerFunction("is not distinct from", SqlStdOperatorTable.IS_NOT_DISTINCT_FROM, hToken(HiveParser.TOK_ISNOTDISTINCTFROM, "TOK_ISNULL")); registerFunction("when", SqlStdOperatorTable.CASE, hToken(HiveParser.Identifier, "when")); registerDuplicateFunction("case", SqlStdOperatorTable.CASE, hToken(HiveParser.Identifier, "when")); // timebased diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g b/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g index d98a663..108b714 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g @@ -397,6 +397,8 @@ TOK_OPERATOR; TOK_EXPRESSION; TOK_DETAIL; TOK_BLOCKING; +TOK_ISDISTINCTFROM; +TOK_ISNOTDISTINCTFROM; } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g b/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g index 8c4ee8a..64e9120 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g @@ -585,9 +585,16 @@ precedenceSimilarExpressionPartNot[CommonTree t] precedenceSimilarExpressionAtom[$t] ; +isDistinctFrom + : + KW_IS (a=KW_NOT)? KW_DISTINCT KW_FROM + -> {$a !=null}? ^(TOK_ISNOTDISTINCTFROM) + -> ^(TOK_ISDISTINCTFROM) + ; + precedenceEqualOperator : - EQUAL | EQUAL_NS | NOTEQUAL + EQUAL | EQUAL_NS | NOTEQUAL | isDistinctFrom ; precedenceEqualExpression diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java index f979c14..107c92e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java @@ -715,6 +715,15 @@ public ColumnExprProcessor getColumnExprProcessor() { static HashMap specialUnaryOperatorTextHashMap; static HashMap specialFunctionTextHashMap; + // this hash map maps ast tokens (added in grammar) which are transformed + // into UDFs but aren't function oar unary operator + // e.g. is distinct from, is not distinct from + // reason we keep this map is because this name is registered in function registry + // so that show functions or describe functions display 'is distinct from' instead of + // TOK_DISTINCT_FROM + // Make sure the name is same as used in Fuction Registry + // getFuctionText uses this map to lookup for name + static HashMap specialExpressiontoTextHashMap; static HashMap conversionFunctionTextHashMap; static HashSet windowingTokens; static { @@ -724,6 +733,9 @@ public ColumnExprProcessor getColumnExprProcessor() { specialFunctionTextHashMap = new HashMap(); specialFunctionTextHashMap.put(HiveParser.TOK_ISNULL, "isnull"); specialFunctionTextHashMap.put(HiveParser.TOK_ISNOTNULL, "isnotnull"); + specialExpressiontoTextHashMap = new HashMap(); + specialExpressiontoTextHashMap.put(HiveParser.TOK_ISNOTDISTINCTFROM, "is not distinct from"); + specialExpressiontoTextHashMap.put(HiveParser.TOK_ISDISTINCTFROM, "is distinct from"); conversionFunctionTextHashMap = new HashMap(); conversionFunctionTextHashMap.put(HiveParser.TOK_BOOLEAN, serdeConstants.BOOLEAN_TYPE_NAME); @@ -807,6 +819,9 @@ public static String getFunctionText(ASTNode expr, boolean isFunction) { funcText = specialUnaryOperatorTextHashMap.get(expr.getType()); } if (funcText == null) { + funcText = specialExpressiontoTextHashMap.get(expr.getType()); + } + if(funcText == null) { funcText = expr.getText(); } } else { diff --git a/ql/src/test/queries/clientpositive/is_distinct_from.q b/ql/src/test/queries/clientpositive/is_distinct_from.q new file mode 100644 index 0000000..e62f48b --- /dev/null +++ b/ql/src/test/queries/clientpositive/is_distinct_from.q @@ -0,0 +1,23 @@ +explain select 1 is distinct from 1, + 1 is distinct from 2, + 1 is distinct from null, + null is distinct from null + from part; + +select 1 is distinct from 1, + 1 is distinct from 2, + 1 is distinct from null, + null is distinct from null + from part; + +explain select 1 is not distinct from 1, + 1 is not distinct from 2, + 1 is not distinct from null, + null is not distinct from null + from part; + +select 1 is not distinct from 1, + 1 is not distinct from 2, + 1 is not distinct from null, + null is not distinct from null + from part; diff --git a/ql/src/test/results/clientpositive/llap/is_distinct_from.q.out b/ql/src/test/results/clientpositive/llap/is_distinct_from.q.out new file mode 100644 index 0000000..14cc130 --- /dev/null +++ b/ql/src/test/results/clientpositive/llap/is_distinct_from.q.out @@ -0,0 +1,138 @@ +PREHOOK: query: explain select 1 is distinct from 1, + 1 is distinct from 2, + 1 is distinct from null, + null is distinct from null + from part +PREHOOK: type: QUERY +POSTHOOK: query: explain select 1 is distinct from 1, + 1 is distinct from 2, + 1 is distinct from null, + null is distinct from null + from part +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: part + Select Operator + expressions: false (type: boolean), true (type: boolean), true (type: boolean), false (type: boolean) + outputColumnNames: _col0, _col1, _col2, _col3 + ListSink + +PREHOOK: query: select 1 is distinct from 1, + 1 is distinct from 2, + 1 is distinct from null, + null is distinct from null + from part +PREHOOK: type: QUERY +PREHOOK: Input: default@part +#### A masked pattern was here #### +POSTHOOK: query: select 1 is distinct from 1, + 1 is distinct from 2, + 1 is distinct from null, + null is distinct from null + from part +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part +#### A masked pattern was here #### +false true true false +false true true false +false true true false +false true true false +false true true false +false true true false +false true true false +false true true false +false true true false +false true true false +false true true false +false true true false +false true true false +false true true false +false true true false +false true true false +false true true false +false true true false +false true true false +false true true false +false true true false +false true true false +false true true false +false true true false +false true true false +false true true false +PREHOOK: query: explain select 1 is not distinct from 1, + 1 is not distinct from 2, + 1 is not distinct from null, + null is not distinct from null + from part +PREHOOK: type: QUERY +POSTHOOK: query: explain select 1 is not distinct from 1, + 1 is not distinct from 2, + 1 is not distinct from null, + null is not distinct from null + from part +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: part + Select Operator + expressions: true (type: boolean), false (type: boolean), false (type: boolean), true (type: boolean) + outputColumnNames: _col0, _col1, _col2, _col3 + ListSink + +PREHOOK: query: select 1 is not distinct from 1, + 1 is not distinct from 2, + 1 is not distinct from null, + null is not distinct from null + from part +PREHOOK: type: QUERY +PREHOOK: Input: default@part +#### A masked pattern was here #### +POSTHOOK: query: select 1 is not distinct from 1, + 1 is not distinct from 2, + 1 is not distinct from null, + null is not distinct from null + from part +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part +#### A masked pattern was here #### +true false false true +true false false true +true false false true +true false false true +true false false true +true false false true +true false false true +true false false true +true false false true +true false false true +true false false true +true false false true +true false false true +true false false true +true false false true +true false false true +true false false true +true false false true +true false false true +true false false true +true false false true +true false false true +true false false true +true false false true +true false false true +true false false true diff --git a/ql/src/test/results/clientpositive/show_functions.q.out b/ql/src/test/results/clientpositive/show_functions.q.out index 68e248a..a355384 100644 --- a/ql/src/test/results/clientpositive/show_functions.q.out +++ b/ql/src/test/results/clientpositive/show_functions.q.out @@ -121,6 +121,8 @@ initcap inline instr internal_interval +is distinct from +is not distinct from isnotnull isnull java_method diff --git a/ql/src/test/results/clientpositive/udf_equal.q.out b/ql/src/test/results/clientpositive/udf_equal.q.out index 52bd843..1c19748 100644 --- a/ql/src/test/results/clientpositive/udf_equal.q.out +++ b/ql/src/test/results/clientpositive/udf_equal.q.out @@ -43,6 +43,7 @@ PREHOOK: type: DESCFUNCTION POSTHOOK: query: DESCRIBE FUNCTION EXTENDED <=> POSTHOOK: type: DESCFUNCTION a <=> b - Returns same result with EQUAL(=) operator for non-null operands, but returns TRUE if both are NULL, FALSE if one of the them is NULL +Synonyms: is not distinct from Function class:org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqualNS Function type:BUILTIN PREHOOK: query: SELECT true<=>false, false<=>true, false<=>false, true<=>true, NULL<=>NULL, true<=>NULL, NULL<=>true, false<=>NULL, NULL<=>false FROM src tablesample (1 rows)