diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveCalciteUtil.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveCalciteUtil.java index 1f8a48c7ad..89ee6c52e3 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveCalciteUtil.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveCalciteUtil.java @@ -69,6 +69,7 @@ import org.apache.hadoop.hive.ql.metadata.VirtualColumn; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveMultiJoin; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject; +import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveSqlFunction; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableFunctionScan; import org.apache.hadoop.hive.ql.optimizer.calcite.translator.ExprNodeConverter; import org.apache.hadoop.hive.ql.optimizer.calcite.translator.SqlFunctionConverter; @@ -978,7 +979,6 @@ public static boolean isMaterializable(RexNode expr) { * Check if the expression is usable for query materialization, returning the first failing expression. */ public static RexCall checkMaterializable(RexNode expr) { - boolean deterministic = true; RexCall failingCall = null; if (expr == null) { @@ -989,7 +989,9 @@ public static RexCall checkMaterializable(RexNode expr) { @Override public Void visitCall(org.apache.calcite.rex.RexCall call) { // non-deterministic functions as well as runtime constants are not materializable. - if (!call.getOperator().isDeterministic() || call.getOperator().isDynamicFunction()) { + SqlOperator op = call.getOperator(); + if (!op.isDeterministic() || op.isDynamicFunction() || + (op instanceof HiveSqlFunction && ((HiveSqlFunction) op).isRuntimeConstant())) { throw new Util.FoundOne(call); } return super.visitCall(call); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveSqlFunction.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveSqlFunction.java new file mode 100644 index 0000000000..a20520bbec --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveSqlFunction.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.optimizer.calcite.reloperators; + +import org.apache.calcite.sql.SqlFunction; +import org.apache.calcite.sql.SqlFunctionCategory; +import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.type.SqlOperandTypeChecker; +import org.apache.calcite.sql.type.SqlOperandTypeInference; +import org.apache.calcite.sql.type.SqlReturnTypeInference; + +public class HiveSqlFunction extends SqlFunction { + private final boolean deterministic; + private final boolean runtimeConstant; + + public HiveSqlFunction(String name, SqlKind kind, SqlReturnTypeInference returnTypeInference, + SqlOperandTypeInference operandTypeInference, SqlOperandTypeChecker operandTypeChecker, + SqlFunctionCategory category, boolean deterministic, boolean runtimeConstant) { + super(name, kind, returnTypeInference, operandTypeInference, operandTypeChecker, category); + this.deterministic = deterministic; + this.runtimeConstant = runtimeConstant; + } + + @Override + public boolean isDeterministic() { + return deterministic; + } + + /** + * Whether it is safe to cache or materialize plans containing this operator. + * We do not rely on {@link SqlFunction#isDynamicFunction()} because it has + * different implications, e.g., a dynamic function will not be reduced in + * Calcite since plans may be cached in the context of prepared statements. + * In our case, we check whether a plan contains runtime constants before + * constant folding happens, hence we can let Calcite reduce these functions. + * + * @return true iff it is unsafe to cache or materialized query plans + * referencing this operator + */ + public boolean isRuntimeConstant() { + return runtimeConstant; + } +} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/SqlFunctionConverter.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/SqlFunctionConverter.java index f8b9fb4d69..a555749fb9 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/SqlFunctionConverter.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/SqlFunctionConverter.java @@ -23,7 +23,6 @@ import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.sql.SqlAggFunction; -import org.apache.calcite.sql.SqlFunction; import org.apache.calcite.sql.SqlFunctionCategory; import org.apache.calcite.sql.SqlKind; import org.apache.calcite.sql.SqlOperator; @@ -58,6 +57,7 @@ import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveFloorDate; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveFromUnixTimeSqlOperator; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveIn; +import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveSqlFunction; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveToDateSqlOperator; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTruncSqlOperator; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveUnixTimestampSqlOperator; @@ -121,10 +121,10 @@ public static SqlOperator getCalciteOperator(String funcTextName, GenericUDF hiv } // For calcite, isDeterministic just matters for within the query. - // isDynamicFunction used to indicate the function is not deterministic between queries. + // runtimeConstant used to indicate the function is not deterministic between queries. boolean isDeterministic = FunctionRegistry.isConsistentWithinQuery(hiveUDF); - boolean isDynamicFunction = FunctionRegistry.isRuntimeConstant(hiveUDF); - return getCalciteFn(name, calciteArgTypes, retType, isDeterministic, isDynamicFunction); + boolean runtimeConstant = FunctionRegistry.isRuntimeConstant(hiveUDF); + return getCalciteFn(name, calciteArgTypes, retType, isDeterministic, runtimeConstant); } public static SqlOperator getCalciteOperator(String funcTextName, GenericUDTF hiveUDTF, @@ -517,29 +517,6 @@ public boolean isDistinct() { } } - private static class CalciteSqlFn extends SqlFunction { - private final boolean deterministic; - private final boolean dynamicFunction; - - public CalciteSqlFn(String name, SqlKind kind, SqlReturnTypeInference returnTypeInference, - SqlOperandTypeInference operandTypeInference, SqlOperandTypeChecker operandTypeChecker, - SqlFunctionCategory category, boolean deterministic, boolean dynamicFunction) { - super(name, kind, returnTypeInference, operandTypeInference, operandTypeChecker, category); - this.deterministic = deterministic; - this.dynamicFunction = dynamicFunction; - } - - @Override - public boolean isDeterministic() { - return deterministic; - } - - @Override - public boolean isDynamicFunction() { - return dynamicFunction; - } - } - private static class CalciteUDFInfo { private String udfName; private SqlReturnTypeInference returnTypeInference; @@ -563,7 +540,7 @@ private static CalciteUDFInfo getUDFInfo(String hiveUdfName, public static SqlOperator getCalciteFn(String hiveUdfName, ImmutableList calciteArgTypes, RelDataType calciteRetType, - boolean deterministic, boolean dynamicFunction) + boolean deterministic, boolean runtimeConstant) throws CalciteSemanticException { if (hiveUdfName != null && hiveUdfName.trim().equals("<=>")) { @@ -589,9 +566,9 @@ public static SqlOperator getCalciteFn(String hiveUdfName, default: calciteOp = hiveToCalcite.get(hiveUdfName); if (null == calciteOp) { - calciteOp = new CalciteSqlFn(uInf.udfName, SqlKind.OTHER_FUNCTION, uInf.returnTypeInference, + calciteOp = new HiveSqlFunction(uInf.udfName, SqlKind.OTHER_FUNCTION, uInf.returnTypeInference, uInf.operandTypeInference, uInf.operandTypeChecker, - SqlFunctionCategory.USER_DEFINED_FUNCTION, deterministic, dynamicFunction); + SqlFunctionCategory.USER_DEFINED_FUNCTION, deterministic, runtimeConstant); } break; } diff --git a/ql/src/test/queries/clientpositive/current_date_timestamp.q b/ql/src/test/queries/clientpositive/current_date_timestamp.q index 8715b97c24..0f90bde7c7 100644 --- a/ql/src/test/queries/clientpositive/current_date_timestamp.q +++ b/ql/src/test/queries/clientpositive/current_date_timestamp.q @@ -5,6 +5,8 @@ select current_timestamp = current_timestamp(), current_date = current_date() fr set hive.test.currenttimestamp =2012-01-01 01:02:03; +explain cbo select current_timestamp() from alltypesorc; + --ensure that timestamp is same for all the rows while using current_timestamp() query should return single row select count(*) from (select current_timestamp() from alltypesorc union select current_timestamp() from src limit 5 ) subq; diff --git a/ql/src/test/results/clientpositive/llap/current_date_timestamp.q.out b/ql/src/test/results/clientpositive/llap/current_date_timestamp.q.out index 280ffb07e1..54244672e4 100644 --- a/ql/src/test/results/clientpositive/llap/current_date_timestamp.q.out +++ b/ql/src/test/results/clientpositive/llap/current_date_timestamp.q.out @@ -11,6 +11,18 @@ true true true true true true true true +PREHOOK: query: explain cbo select current_timestamp() from alltypesorc +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypesorc +#### A masked pattern was here #### +POSTHOOK: query: explain cbo select current_timestamp() from alltypesorc +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alltypesorc +#### A masked pattern was here #### +CBO PLAN: +HiveProject($f0=[CAST(2012-01-01 01:02:03):TIMESTAMP(9)]) + HiveTableScan(table=[[default, alltypesorc]], table:alias=[alltypesorc]) + PREHOOK: query: select count(*) from (select current_timestamp() from alltypesorc union select current_timestamp() from src limit 5 ) subq PREHOOK: type: QUERY PREHOOK: Input: default@alltypesorc @@ -47,7 +59,7 @@ POSTHOOK: query: explain extended select current_timestamp() from alltypesorc POSTHOOK: type: QUERY POSTHOOK: Input: default@alltypesorc #### A masked pattern was here #### -OPTIMIZED SQL: SELECT CURRENT_TIMESTAMP() AS `$f0` +OPTIMIZED SQL: SELECT CAST(TIMESTAMP '2012-01-01 01:02:03.000000000' AS TIMESTAMP(9)) AS `$f0` FROM `default`.`alltypesorc` STAGE DEPENDENCIES: Stage-0 is a root stage