diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties index fb85b9ece2..29102118eb 100644 --- a/itests/src/test/resources/testconfiguration.properties +++ b/itests/src/test/resources/testconfiguration.properties @@ -501,6 +501,7 @@ minillaplocal.query.files=acid_globallimit.q,\ groupby2.q,\ hybridgrace_hashjoin_1.q,\ hybridgrace_hashjoin_2.q,\ + is_distinct_from.q,\ infer_bucket_sort_bucketed_table.q,\ input16_cc.q,\ insert_dir_distcp.q,\ diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/SqlFunctionConverter.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/SqlFunctionConverter.java index 85450c999f..10f5eb3b5b 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/SqlFunctionConverter.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/SqlFunctionConverter.java @@ -352,6 +352,7 @@ private static String getName(GenericUDF hiveUDF) { registerFunction("struct", SqlStdOperatorTable.ROW, hToken(HiveParser.Identifier, "struct")); registerFunction("isnotnull", SqlStdOperatorTable.IS_NOT_NULL, hToken(HiveParser.TOK_ISNOTNULL, "TOK_ISNOTNULL")); registerFunction("isnull", SqlStdOperatorTable.IS_NULL, hToken(HiveParser.TOK_ISNULL, "TOK_ISNULL")); + registerFunction("is not distinct from", SqlStdOperatorTable.IS_NOT_DISTINCT_FROM, hToken(HiveParser.EQUAL_NS, "<=>")); registerFunction("when", SqlStdOperatorTable.CASE, hToken(HiveParser.Identifier, "when")); registerDuplicateFunction("case", SqlStdOperatorTable.CASE, hToken(HiveParser.Identifier, "when")); // timebased diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g b/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g index 8c4ee8a38b..2fff3a1c51 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g @@ -585,14 +585,27 @@ precedenceSimilarExpressionPartNot[CommonTree t] precedenceSimilarExpressionAtom[$t] ; +precedenceDistinctOperator + : + KW_IS KW_DISTINCT KW_FROM + ; + precedenceEqualOperator : - EQUAL | EQUAL_NS | NOTEQUAL + EQUAL | EQUAL_NS | NOTEQUAL | KW_IS KW_NOT KW_DISTINCT KW_FROM -> EQUAL_NS ; precedenceEqualExpression : - precedenceSimilarExpression (precedenceEqualOperator^ precedenceSimilarExpression)* + (precedenceSimilarExpression -> precedenceSimilarExpression) + ( + equal=precedenceEqualOperator p=precedenceSimilarExpression + -> ^($equal {$precedenceEqualExpression.tree} $p) + | + dist=precedenceDistinctOperator p=precedenceSimilarExpression + -> ^(KW_NOT ^(EQUAL_NS {$precedenceEqualExpression.tree} $p)) + )* + -> {$precedenceEqualExpression.tree} ; precedenceNotOperator diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java index f979c1432e..35db097a83 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java @@ -715,6 +715,18 @@ public ColumnExprProcessor getColumnExprProcessor() { static HashMap specialUnaryOperatorTextHashMap; static HashMap specialFunctionTextHashMap; + + // specialExpressiontoTextHashMap maps ast tokens to text + // for tokens which are rewritten in grammar but aren't part of input query + // e.g. 'is distinct from' is rewritten into + // KW_NOT + // EQUAL_NS + // Since these token are formed from rewrite rules they have text 'KW_NOT' and 'EQUAL_NS' + // instead of 'not' and '<=>' + // For this reason we manually map these tokens into appropriate text/name so that + // function lookup (getFunctionText) works appropriately. + static HashMap specialExpressiontoTextHashMap; + static HashMap conversionFunctionTextHashMap; static HashSet windowingTokens; static { @@ -724,6 +736,9 @@ public ColumnExprProcessor getColumnExprProcessor() { specialFunctionTextHashMap = new HashMap(); specialFunctionTextHashMap.put(HiveParser.TOK_ISNULL, "isnull"); specialFunctionTextHashMap.put(HiveParser.TOK_ISNOTNULL, "isnotnull"); + specialExpressiontoTextHashMap = new HashMap(); + specialExpressiontoTextHashMap.put(HiveParser.EQUAL_NS, "<=>"); + specialExpressiontoTextHashMap.put(HiveParser.KW_NOT, "not"); conversionFunctionTextHashMap = new HashMap(); conversionFunctionTextHashMap.put(HiveParser.TOK_BOOLEAN, serdeConstants.BOOLEAN_TYPE_NAME); @@ -807,6 +822,9 @@ public static String getFunctionText(ASTNode expr, boolean isFunction) { funcText = specialUnaryOperatorTextHashMap.get(expr.getType()); } if (funcText == null) { + funcText = specialExpressiontoTextHashMap.get(expr.getType()); + } + if(funcText == null) { funcText = expr.getText(); } } else { diff --git a/ql/src/test/queries/clientpositive/is_distinct_from.q b/ql/src/test/queries/clientpositive/is_distinct_from.q new file mode 100644 index 0000000000..e62f48b044 --- /dev/null +++ b/ql/src/test/queries/clientpositive/is_distinct_from.q @@ -0,0 +1,23 @@ +explain select 1 is distinct from 1, + 1 is distinct from 2, + 1 is distinct from null, + null is distinct from null + from part; + +select 1 is distinct from 1, + 1 is distinct from 2, + 1 is distinct from null, + null is distinct from null + from part; + +explain select 1 is not distinct from 1, + 1 is not distinct from 2, + 1 is not distinct from null, + null is not distinct from null + from part; + +select 1 is not distinct from 1, + 1 is not distinct from 2, + 1 is not distinct from null, + null is not distinct from null + from part; diff --git a/ql/src/test/results/clientpositive/llap/is_distinct_from.q.out b/ql/src/test/results/clientpositive/llap/is_distinct_from.q.out new file mode 100644 index 0000000000..14cc130be0 --- /dev/null +++ b/ql/src/test/results/clientpositive/llap/is_distinct_from.q.out @@ -0,0 +1,138 @@ +PREHOOK: query: explain select 1 is distinct from 1, + 1 is distinct from 2, + 1 is distinct from null, + null is distinct from null + from part +PREHOOK: type: QUERY +POSTHOOK: query: explain select 1 is distinct from 1, + 1 is distinct from 2, + 1 is distinct from null, + null is distinct from null + from part +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: part + Select Operator + expressions: false (type: boolean), true (type: boolean), true (type: boolean), false (type: boolean) + outputColumnNames: _col0, _col1, _col2, _col3 + ListSink + +PREHOOK: query: select 1 is distinct from 1, + 1 is distinct from 2, + 1 is distinct from null, + null is distinct from null + from part +PREHOOK: type: QUERY +PREHOOK: Input: default@part +#### A masked pattern was here #### +POSTHOOK: query: select 1 is distinct from 1, + 1 is distinct from 2, + 1 is distinct from null, + null is distinct from null + from part +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part +#### A masked pattern was here #### +false true true false +false true true false +false true true false +false true true false +false true true false +false true true false +false true true false +false true true false +false true true false +false true true false +false true true false +false true true false +false true true false +false true true false +false true true false +false true true false +false true true false +false true true false +false true true false +false true true false +false true true false +false true true false +false true true false +false true true false +false true true false +false true true false +PREHOOK: query: explain select 1 is not distinct from 1, + 1 is not distinct from 2, + 1 is not distinct from null, + null is not distinct from null + from part +PREHOOK: type: QUERY +POSTHOOK: query: explain select 1 is not distinct from 1, + 1 is not distinct from 2, + 1 is not distinct from null, + null is not distinct from null + from part +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: part + Select Operator + expressions: true (type: boolean), false (type: boolean), false (type: boolean), true (type: boolean) + outputColumnNames: _col0, _col1, _col2, _col3 + ListSink + +PREHOOK: query: select 1 is not distinct from 1, + 1 is not distinct from 2, + 1 is not distinct from null, + null is not distinct from null + from part +PREHOOK: type: QUERY +PREHOOK: Input: default@part +#### A masked pattern was here #### +POSTHOOK: query: select 1 is not distinct from 1, + 1 is not distinct from 2, + 1 is not distinct from null, + null is not distinct from null + from part +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part +#### A masked pattern was here #### +true false false true +true false false true +true false false true +true false false true +true false false true +true false false true +true false false true +true false false true +true false false true +true false false true +true false false true +true false false true +true false false true +true false false true +true false false true +true false false true +true false false true +true false false true +true false false true +true false false true +true false false true +true false false true +true false false true +true false false true +true false false true +true false false true