partToPruner = null;
ExprNodeDesc pruner_pred = null;
diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/SharedWorkOptimizer.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/SharedWorkOptimizer.java
index 2f49985800..caed527d71 100644
--- ql/src/java/org/apache/hadoop/hive/ql/optimizer/SharedWorkOptimizer.java
+++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/SharedWorkOptimizer.java
@@ -487,6 +487,15 @@ private static boolean sharedWorkOptimization(ParseContext pctx, SharedWorkOptim
}
LOG.debug("Input operator removed: {}", op);
}
+
+ // A shared TSop across branches can not have probeContext that utilizes single branch info
+ // Filtered-out rows from one branch might be needed by another branch sharing a TSop
+ if (retainableTsOp.getProbeDecodeContext() != null) {
+ LOG.debug("Removing probeDecodeCntx for merged TS op {}", retainableTsOp);
+ retainableTsOp.setProbeDecodeContext(null);
+ retainableTsOp.getConf().setProbeDecodeContext(null);
+ }
+
// Then we merge the operators of the works we are going to merge
optimizerCache.removeOpAndCombineWork(discardableTsOp, retainableTsOp);
removedOps.add(discardableTsOp);
diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveCalciteUtil.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveCalciteUtil.java
index 4a18cfef54..636c3a2ebb 100644
--- ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveCalciteUtil.java
+++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveCalciteUtil.java
@@ -49,6 +49,7 @@
import org.apache.calcite.rex.RexNode;
import org.apache.calcite.rex.RexOver;
import org.apache.calcite.rex.RexPatternFieldRef;
+import org.apache.calcite.rex.RexShuttle;
import org.apache.calcite.rex.RexTableInputRef;
import org.apache.calcite.rex.RexRangeRef;
import org.apache.calcite.rex.RexSubQuery;
@@ -59,6 +60,8 @@
import org.apache.calcite.sql.SqlKind;
import org.apache.calcite.sql.SqlOperator;
import org.apache.calcite.sql.fun.SqlStdOperatorTable;
+import org.apache.calcite.sql.type.SqlTypeFamily;
+import org.apache.calcite.sql.type.SqlTypeName;
import org.apache.calcite.sql.validate.SqlValidatorUtil;
import org.apache.calcite.util.ImmutableBitSet;
import org.apache.calcite.util.Pair;
@@ -80,8 +83,6 @@
import org.apache.hadoop.hive.ql.parse.SemanticException;
import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import com.google.common.base.Function;
import com.google.common.collect.ImmutableList;
@@ -98,9 +99,6 @@
public class HiveCalciteUtil {
- private static final Logger LOG = LoggerFactory.getLogger(HiveCalciteUtil.class);
-
-
/**
* Get list of virtual columns from the given list of projections.
*
@@ -1062,6 +1060,25 @@ public RexNode apply(RelDataTypeField input) {
return HiveProject.create(input, copyInputRefs, null);
}
+ public static boolean isLiteral(RexNode expr) {
+ if (expr instanceof RexCall) {
+ RexCall call = (RexCall) expr;
+ if (call.getOperator() == SqlStdOperatorTable.ROW ||
+ call.getOperator() == SqlStdOperatorTable.ARRAY_VALUE_CONSTRUCTOR ||
+ call.getOperator() == SqlStdOperatorTable.MAP_VALUE_CONSTRUCTOR) {
+ // We check all operands
+ for (RexNode node : call.getOperands()) {
+ if (!isLiteral(node)) {
+ return false;
+ }
+ }
+ // All literals
+ return true;
+ }
+ }
+ return expr.isA(SqlKind.LITERAL);
+ }
+
/**
* Walks over an expression and determines whether it is constant.
*/
@@ -1157,4 +1174,71 @@ public Void visitInputRef(RexInputRef inputRef) {
return inputRefSet;
}
}
+
+ /** Fixes up the type of all {@link RexInputRef}s in an
+ * expression to match differences in nullability.
+ *
+ * This can be useful in case a field is inferred to be not nullable,
+ * e.g., a not null literal, and the reference to the row type needs
+ * to be changed to adjust the nullability flag.
+ *
+ * In case of references created on top of a Calcite schema generated
+ * directly from a Hive schema, this is especially useful since Hive
+ * does not have a notion of nullability so all fields in the schema
+ * will be inferred to nullable. However, Calcite makes this distinction.
+ *
+ *
Throws if there any greater inconsistencies of type. */
+ public static List fixNullability(final RexBuilder rexBuilder,
+ List nodes, final List fieldTypes) {
+ return new FixNullabilityShuttle(rexBuilder, fieldTypes).apply(nodes);
+ }
+
+ /** Fixes up the type of all {@link RexInputRef}s in an
+ * expression to match differences in nullability.
+ *
+ * Throws if there any greater inconsistencies of type. */
+ public static RexNode fixNullability(final RexBuilder rexBuilder,
+ RexNode node, final List fieldTypes) {
+ return new FixNullabilityShuttle(rexBuilder, fieldTypes).apply(node);
+ }
+
+ /** Shuttle that fixes up an expression to match changes in nullability of
+ * input fields. */
+ public static class FixNullabilityShuttle extends RexShuttle {
+ private final List typeList;
+ private final RexBuilder rexBuilder;
+
+ public FixNullabilityShuttle(RexBuilder rexBuilder,
+ List typeList) {
+ this.typeList = typeList;
+ this.rexBuilder = rexBuilder;
+ }
+
+ @Override public RexNode visitInputRef(RexInputRef ref) {
+ final RelDataType rightType = typeList.get(ref.getIndex());
+ final RelDataType refType = ref.getType();
+ if (refType == rightType) {
+ return ref;
+ }
+ final RelDataType refType2 =
+ rexBuilder.getTypeFactory().createTypeWithNullability(refType,
+ rightType.isNullable());
+ // This is a validation check which can become quite handy debugging type
+ // issues. Basically, we need both types to be equal, only difference should
+ // be nullability.
+ // However, we make an exception for Hive wrt CHAR type because Hive encodes
+ // the STRING type for literals within CHAR value (see {@link HiveNlsString})
+ // while Calcite always considers these literals to be a CHAR, which means
+ // that the reference may be created as a STRING or VARCHAR from AST node
+ // at parsing time but the actual type referenced is a CHAR.
+ if (refType2 == rightType) {
+ return new RexInputRef(ref.getIndex(), refType2);
+ } else if (refType2.getFamily() == SqlTypeFamily.CHARACTER &&
+ rightType.getSqlTypeName() == SqlTypeName.CHAR && !rightType.isNullable()) {
+ return new RexInputRef(ref.getIndex(), rightType);
+ }
+ throw new AssertionError("mismatched type " + ref + " " + rightType);
+ }
+ }
+
}
diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRexExecutorImpl.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRexExecutorImpl.java
index 1dede0f88a..08b4e8db4b 100644
--- ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRexExecutorImpl.java
+++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRexExecutorImpl.java
@@ -15,44 +15,43 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.hadoop.hive.ql.optimizer.calcite;
import java.util.HashSet;
import java.util.List;
-import org.apache.calcite.plan.RelOptCluster;
import org.apache.calcite.rex.RexBuilder;
import org.apache.calcite.rex.RexExecutorImpl;
import org.apache.calcite.rex.RexNode;
import org.apache.hadoop.hive.ql.optimizer.ConstantPropagateProcFactory;
import org.apache.hadoop.hive.ql.optimizer.calcite.translator.ExprNodeConverter;
import org.apache.hadoop.hive.ql.optimizer.calcite.translator.RexNodeConverter;
+import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-
+/**
+ * Executor for {@link RexNode} based on Hive semantics.
+ */
public class HiveRexExecutorImpl extends RexExecutorImpl {
private static final Logger LOG = LoggerFactory.getLogger(HiveRexExecutorImpl.class);
- private final RelOptCluster cluster;
- public HiveRexExecutorImpl(RelOptCluster cluster) {
+ public HiveRexExecutorImpl() {
super(null);
- this.cluster = cluster;
}
@Override
public void reduce(RexBuilder rexBuilder, List constExps, List reducedValues) {
- RexNodeConverter rexNodeConverter = new RexNodeConverter(cluster);
+ RexNodeConverter rexNodeConverter = new RexNodeConverter(rexBuilder, rexBuilder.getTypeFactory());
for (RexNode rexNode : constExps) {
// initialize the converter
ExprNodeConverter converter = new ExprNodeConverter("", null, null, null,
- new HashSet(), cluster.getTypeFactory());
+ new HashSet<>(), rexBuilder.getTypeFactory());
// convert RexNode to ExprNodeGenericFuncDesc
ExprNodeDesc expr = rexNode.accept(converter);
if (expr instanceof ExprNodeGenericFuncDesc) {
@@ -60,20 +59,27 @@ public void reduce(RexBuilder rexBuilder, List constExps, List
ExprNodeDesc constant = ConstantPropagateProcFactory
.foldExpr((ExprNodeGenericFuncDesc) expr);
if (constant != null) {
- try {
- // convert constant back to RexNode
- reducedValues.add(rexNodeConverter.convert(constant));
- } catch (Exception e) {
- LOG.warn(e.getMessage());
- reducedValues.add(rexNode);
- }
+ addExpressionToList(constant, rexNode, rexNodeConverter, reducedValues);
} else {
reducedValues.add(rexNode);
}
+ } else if (expr instanceof ExprNodeConstantDesc) {
+ addExpressionToList(expr, rexNode, rexNodeConverter, reducedValues);
} else {
reducedValues.add(rexNode);
}
}
}
+ private void addExpressionToList(ExprNodeDesc reducedExpr, RexNode originalExpr,
+ RexNodeConverter rexNodeConverter, List reducedValues) {
+ try {
+ // convert constant back to RexNode
+ reducedValues.add(rexNodeConverter.convert(reducedExpr));
+ } catch (Exception e) {
+ LOG.warn(e.getMessage());
+ reducedValues.add(originalExpr);
+ }
+ }
+
}
diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveTypeSystemImpl.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveTypeSystemImpl.java
index 4a5c9cc567..5e301fdbcb 100644
--- ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveTypeSystemImpl.java
+++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveTypeSystemImpl.java
@@ -78,6 +78,7 @@ public int getDefaultPrecision(SqlTypeName typeName) {
// Binary doesn't need any sizes; Decimal has the default of 10.
case BINARY:
case VARBINARY:
+ return RelDataType.PRECISION_NOT_SPECIFIED;
case TIME:
case TIMESTAMP:
case TIMESTAMP_WITH_LOCAL_TIME_ZONE:
diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveRexExprList.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveRexExprList.java
new file mode 100644
index 0000000000..707bcca0b5
--- /dev/null
+++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveRexExprList.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.optimizer.calcite.reloperators;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rex.RexBiVisitor;
+import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.rex.RexVisitor;
+import org.apache.hadoop.hive.ql.plan.ExprNodeColumnListDesc;
+
+/**
+ * This class represents the equivalent to {@link ExprNodeColumnListDesc}
+ * in a Calcite expression. It is not supposed to be used through planning
+ * and should be immediately expanded after it has been generated by
+ * the parser.
+ */
+public class HiveRexExprList extends RexNode {
+
+ final List expressions = new ArrayList<>();
+
+ public void addExpression(RexNode expression) {
+ expressions.add(expression);
+ }
+
+ public List getExpressions() {
+ return expressions;
+ }
+
+ @Override
+ public RelDataType getType() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public R accept(RexVisitor visitor) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public R accept(RexBiVisitor visitor, P arg) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj instanceof HiveRexExprList) {
+ return this.expressions.equals(((HiveRexExprList) obj).expressions);
+ }
+ return false;
+ }
+
+ @Override
+ public int hashCode() {
+ return expressions.hashCode();
+ }
+
+}
diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveToUnixTimestampSqlOperator.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveToUnixTimestampSqlOperator.java
new file mode 100644
index 0000000000..b5ab01c9ad
--- /dev/null
+++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveToUnixTimestampSqlOperator.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.optimizer.calcite.reloperators;
+
+import org.apache.calcite.sql.fun.SqlAbstractTimeFunction;
+import org.apache.calcite.sql.type.SqlTypeName;
+
+/**
+ * Sql UNIX_TIMESTAMP calcite operator.
+ */
+public class HiveToUnixTimestampSqlOperator extends SqlAbstractTimeFunction {
+ public static final HiveToUnixTimestampSqlOperator INSTANCE = new HiveToUnixTimestampSqlOperator();
+ protected HiveToUnixTimestampSqlOperator() {
+ super("UNIX_TIMESTAMP", SqlTypeName.BIGINT);
+ }
+}
diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterSortPredicates.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterSortPredicates.java
index 3b3557063e..780481f2fd 100644
--- ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterSortPredicates.java
+++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterSortPredicates.java
@@ -33,6 +33,7 @@
import org.apache.calcite.rex.RexNode;
import org.apache.calcite.rex.RexShuttle;
import org.apache.calcite.rex.RexVisitorImpl;
+import org.apache.calcite.sql.SqlKind;
import org.apache.calcite.util.Pair;
import org.apache.hadoop.hive.ql.optimizer.calcite.stats.FilterSelectivityEstimator;
import org.apache.hadoop.hive.ql.optimizer.calcite.stats.HiveRelMdSize;
@@ -223,7 +224,13 @@ public Double visitCall(RexCall call) {
return null;
}
cost += operandCost;
- Double size = HiveRelMdSize.averageTypeSize(operand.getType());
+ Double size;
+ if (operand.isA(SqlKind.LITERAL)) {
+ size = HiveRelMdSize.INSTANCE.typeValueSize(operand.getType(),
+ ((RexLiteral) operand).getValueAs(Comparable.class));
+ } else {
+ size = HiveRelMdSize.INSTANCE.averageTypeValueSize(operand.getType());
+ }
if (size == null) {
return null;
}
diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveJoinConstraintsRule.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveJoinConstraintsRule.java
index 05c7368c42..8acecfccc2 100644
--- ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveJoinConstraintsRule.java
+++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveJoinConstraintsRule.java
@@ -39,6 +39,7 @@
import org.apache.calcite.util.mapping.Mapping;
import org.apache.calcite.util.mapping.MappingType;
import org.apache.calcite.util.mapping.Mappings;
+import org.apache.hadoop.hive.ql.optimizer.calcite.HiveCalciteUtil;
import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelFactories;
import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelOptUtil;
import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelOptUtil.RewritablePKFKJoinInfo;
@@ -222,7 +223,7 @@ public void onMatch(RelOptRuleCall call) {
.collect(Collectors.toList());
}
// Fix nullability in references to the input node
- topProjExprs = RexUtil.fixUp(rexBuilder, topProjExprs, RelOptUtil.getFieldTypeList(fkInput.getRowType()));
+ topProjExprs = HiveCalciteUtil.fixNullability(rexBuilder, topProjExprs, RelOptUtil.getFieldTypeList(fkInput.getRowType()));
// Trigger transformation
if (nullableNodes.isEmpty()) {
call.transformTo(call.builder()
diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/jdbc/JDBCExpandExpressionsRule.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/jdbc/JDBCExpandExpressionsRule.java
index 1c8a100219..c903bba4e3 100644
--- ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/jdbc/JDBCExpandExpressionsRule.java
+++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/jdbc/JDBCExpandExpressionsRule.java
@@ -35,6 +35,8 @@
import org.apache.calcite.rex.RexUtil;
import org.apache.calcite.sql.SqlKind;
import org.apache.calcite.sql.fun.SqlStdOperatorTable;
+import org.apache.hadoop.hive.ql.optimizer.calcite.HiveCalciteUtil;
+import org.apache.hadoop.hive.ql.optimizer.calcite.translator.RexNodeConverter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -196,24 +198,11 @@ private RexNode transformIntoOrAndClause(RexBuilder rexBuilder, RexCall expressi
return expression;
}
- final List disjuncts = new ArrayList<>(expression.getOperands().size() - 2);
- final RexCall columnExpressions = (RexCall) expression.getOperands().get(0);
- for (int i = 1; i < expression.getOperands().size(); i++) {
- List conjuncts = new ArrayList<>(columnExpressions.getOperands().size() - 1);
- RexCall valueExpressions = (RexCall) expression.getOperands().get(i);
- for (int j = 0; j < columnExpressions.getOperands().size(); j++) {
- conjuncts.add(rexBuilder.makeCall(
- SqlStdOperatorTable.EQUALS,
- columnExpressions.getOperands().get(j),
- valueExpressions.getOperands().get(j)));
- }
- if (conjuncts.size() > 1) {
- disjuncts.add(rexBuilder.makeCall(
- SqlStdOperatorTable.AND,
- conjuncts));
- } else {
- disjuncts.add(conjuncts.get(0));
- }
+ final List disjuncts = RexNodeConverter.transformInToOrOperands(
+ expression.getOperands(), rexBuilder);
+ if (disjuncts == null) {
+ // We could not execute transformation, return expression
+ return expression;
}
if (disjuncts.size() > 1) {
diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdSize.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdSize.java
index eebeb4c67e..d5a4e3636a 100644
--- ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdSize.java
+++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdSize.java
@@ -38,7 +38,7 @@
public class HiveRelMdSize extends RelMdSize {
- private static final HiveRelMdSize INSTANCE = new HiveRelMdSize();
+ public static final HiveRelMdSize INSTANCE = new HiveRelMdSize();
public static final RelMetadataProvider SOURCE =
ReflectiveRelMetadataProvider.reflectiveSource(INSTANCE,
@@ -129,10 +129,6 @@ private HiveRelMdSize() {}
// supports all types
@Override
public Double averageTypeValueSize(RelDataType type) {
- return averageTypeSize(type);
- }
-
- public static Double averageTypeSize(RelDataType type) {
switch (type.getSqlTypeName()) {
case BOOLEAN:
case TINYINT:
@@ -145,10 +141,14 @@ public static Double averageTypeSize(RelDataType type) {
case DECIMAL:
case DATE:
case TIME:
+ case INTERVAL_YEAR:
+ case INTERVAL_YEAR_MONTH:
+ case INTERVAL_MONTH:
return 4d;
case BIGINT:
case DOUBLE:
case TIMESTAMP:
+ case TIMESTAMP_WITH_LOCAL_TIME_ZONE:
case INTERVAL_DAY:
case INTERVAL_DAY_HOUR:
case INTERVAL_DAY_MINUTE:
@@ -158,10 +158,7 @@ public static Double averageTypeSize(RelDataType type) {
case INTERVAL_HOUR_SECOND:
case INTERVAL_MINUTE:
case INTERVAL_MINUTE_SECOND:
- case INTERVAL_MONTH:
case INTERVAL_SECOND:
- case INTERVAL_YEAR:
- case INTERVAL_YEAR_MONTH:
return 8d;
case BINARY:
return (double) type.getPrecision();
@@ -175,12 +172,11 @@ public static Double averageTypeSize(RelDataType type) {
case ROW:
Double average = 0.0;
for (RelDataTypeField field : type.getFieldList()) {
- average += averageTypeSize(field.getType());
+ average += averageTypeValueSize(field.getType());
}
return average;
default:
return null;
}
}
-
}
diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ExprNodeConverter.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ExprNodeConverter.java
index cf104af00b..658d2cccb9 100644
--- ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ExprNodeConverter.java
+++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ExprNodeConverter.java
@@ -23,6 +23,7 @@
import java.util.List;
import java.util.Set;
+import org.apache.calcite.avatica.util.ByteString;
import org.apache.calcite.rel.RelFieldCollation;
import org.apache.calcite.rel.type.RelDataType;
import org.apache.calcite.rel.type.RelDataTypeFactory;
@@ -58,7 +59,6 @@
import org.apache.hadoop.hive.ql.optimizer.ConstantPropagateProcFactory;
import org.apache.hadoop.hive.ql.optimizer.calcite.translator.ASTConverter.RexVisitor;
import org.apache.hadoop.hive.ql.optimizer.calcite.translator.ASTConverter.Schema;
-import org.apache.hadoop.hive.ql.optimizer.calcite.translator.RexNodeConverter.HiveNlsString;
import org.apache.hadoop.hive.ql.parse.ASTNode;
import org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.NullOrder;
import org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.Order;
@@ -73,6 +73,7 @@
import org.apache.hadoop.hive.ql.parse.WindowingSpec.WindowFunctionSpec;
import org.apache.hadoop.hive.ql.parse.WindowingSpec.WindowSpec;
import org.apache.hadoop.hive.ql.parse.WindowingSpec.WindowType;
+import org.apache.hadoop.hive.ql.parse.type.RexNodeExprFactory.HiveNlsString;
import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
@@ -227,6 +228,10 @@ public ExprNodeDesc visitCall(RexCall call) {
@Override
public ExprNodeDesc visitLiteral(RexLiteral literal) {
+ return toExprNodeConstantDesc(literal);
+ }
+
+ public static ExprNodeConstantDesc toExprNodeConstantDesc(RexLiteral literal) {
RelDataType lType = literal.getType();
if (RexLiteral.value(literal) == null) {
@@ -259,12 +264,12 @@ public ExprNodeDesc visitLiteral(RexLiteral literal) {
throw new RuntimeException(e);
}
return new ExprNodeConstantDesc(
- TypeInfoFactory.getTimestampTZTypeInfo(conf.getLocalTimeZone()), null);
+ TypeInfoFactory.getTimestampTZTypeInfo(conf.getLocalTimeZone()), null);
case BINARY:
return new ExprNodeConstantDesc(TypeInfoFactory.binaryTypeInfo, null);
case DECIMAL:
return new ExprNodeConstantDesc(
- TypeInfoFactory.getDecimalTypeInfo(lType.getPrecision(), lType.getScale()), null);
+ TypeInfoFactory.getDecimalTypeInfo(lType.getPrecision(), lType.getScale()), null);
case VARCHAR:
case CHAR:
return new ExprNodeConstantDesc(TypeInfoFactory.stringTypeInfo, null);
@@ -331,7 +336,8 @@ public ExprNodeDesc visitLiteral(RexLiteral literal) {
return new ExprNodeConstantDesc(TypeInfoFactory.getTimestampTZTypeInfo(conf.getLocalTimeZone()),
TimestampTZUtil.parse(literal.getValueAs(TimestampString.class).toString() + " UTC"));
case BINARY:
- return new ExprNodeConstantDesc(TypeInfoFactory.binaryTypeInfo, literal.getValue3());
+ return new ExprNodeConstantDesc(TypeInfoFactory.binaryTypeInfo,
+ literal.getValueAs(ByteString.class).getBytes());
case DECIMAL:
return new ExprNodeConstantDesc(TypeInfoFactory.getDecimalTypeInfo(lType.getPrecision(),
lType.getScale()), HiveDecimal.create((BigDecimal)literal.getValue3()));
@@ -361,7 +367,7 @@ public ExprNodeDesc visitLiteral(RexLiteral literal) {
case INTERVAL_YEAR_MONTH: {
BigDecimal monthsBd = (BigDecimal) literal.getValue();
return new ExprNodeConstantDesc(TypeInfoFactory.intervalYearMonthTypeInfo,
- new HiveIntervalYearMonth(monthsBd.intValue()));
+ new HiveIntervalYearMonth(monthsBd.intValue()));
}
case INTERVAL_DAY:
case INTERVAL_DAY_HOUR:
@@ -377,7 +383,7 @@ public ExprNodeDesc visitLiteral(RexLiteral literal) {
// Calcite literal is in millis, we need to convert to seconds
BigDecimal secsBd = millisBd.divide(BigDecimal.valueOf(1000));
return new ExprNodeConstantDesc(TypeInfoFactory.intervalDayTimeTypeInfo,
- new HiveIntervalDayTime(secsBd));
+ new HiveIntervalDayTime(secsBd));
}
default:
return new ExprNodeConstantDesc(TypeInfoFactory.voidTypeInfo, literal.getValue3());
diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/RexNodeConverter.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/RexNodeConverter.java
index ba6eefb2d2..bd1c84bcc1 100644
--- ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/RexNodeConverter.java
+++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/RexNodeConverter.java
@@ -19,36 +19,27 @@
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableList.Builder;
-import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
import org.apache.calcite.avatica.util.TimeUnit;
import org.apache.calcite.avatica.util.TimeUnitRange;
-import org.apache.calcite.plan.RelOptCluster;
-import org.apache.calcite.rel.RelNode;
-import org.apache.calcite.rel.core.CorrelationId;
import org.apache.calcite.rel.type.RelDataType;
import org.apache.calcite.rel.type.RelDataTypeFactory;
import org.apache.calcite.rex.RexBuilder;
import org.apache.calcite.rex.RexCall;
import org.apache.calcite.rex.RexNode;
-import org.apache.calcite.rex.RexSubQuery;
import org.apache.calcite.rex.RexUtil;
import org.apache.calcite.sql.SqlBinaryOperator;
-import org.apache.calcite.sql.SqlCollation;
import org.apache.calcite.sql.SqlIntervalQualifier;
import org.apache.calcite.sql.SqlKind;
import org.apache.calcite.sql.SqlOperator;
import org.apache.calcite.sql.fun.SqlCastFunction;
import org.apache.calcite.sql.fun.SqlStdOperatorTable;
-import org.apache.calcite.sql.fun.SqlQuantifyOperator;
import org.apache.calcite.sql.parser.SqlParserPos;
import org.apache.calcite.sql.type.SqlTypeName;
import org.apache.calcite.sql.type.SqlTypeUtil;
-import org.apache.calcite.util.ConversionUtil;
import org.apache.calcite.util.DateString;
-import org.apache.calcite.util.NlsString;
import org.apache.calcite.util.TimestampString;
import org.apache.hadoop.hive.common.type.Date;
import org.apache.hadoop.hive.common.type.Decimal128;
@@ -59,28 +50,22 @@
import org.apache.hadoop.hive.common.type.HiveVarchar;
import org.apache.hadoop.hive.common.type.Timestamp;
import org.apache.hadoop.hive.common.type.TimestampTZ;
-import org.apache.hadoop.hive.ql.ErrorMsg;
import org.apache.hadoop.hive.ql.exec.FunctionRegistry;
import org.apache.hadoop.hive.ql.optimizer.calcite.CalciteSemanticException;
import org.apache.hadoop.hive.ql.optimizer.calcite.CalciteSemanticException.UnsupportedFeature;
-import org.apache.hadoop.hive.ql.optimizer.calcite.CalciteSubquerySemanticException;
+import org.apache.hadoop.hive.ql.optimizer.calcite.HiveCalciteUtil;
import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveExtractDate;
import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveFloorDate;
import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveToDateSqlOperator;
-import org.apache.hadoop.hive.ql.optimizer.calcite.translator.RexNodeConverter.HiveNlsString.Interpretation;
-import org.apache.hadoop.hive.ql.parse.ASTNode;
-import org.apache.hadoop.hive.ql.parse.HiveParser;
-import org.apache.hadoop.hive.ql.parse.RowResolver;
import org.apache.hadoop.hive.ql.parse.SemanticException;
import org.apache.hadoop.hive.ql.parse.type.ExprNodeTypeCheck;
-import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc;
+import org.apache.hadoop.hive.ql.parse.type.RexNodeExprFactory;
+import org.apache.hadoop.hive.ql.parse.type.RexNodeExprFactory.HiveNlsString.Interpretation;
import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeDescUtils;
import org.apache.hadoop.hive.ql.plan.ExprNodeFieldDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc;
-import org.apache.hadoop.hive.ql.plan.ExprNodeSubQueryDesc;
-import org.apache.hadoop.hive.ql.plan.SubqueryType;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFBaseBinary;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFBaseCompare;
@@ -110,77 +95,27 @@
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
import java.math.BigDecimal;
-import java.math.BigInteger;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Calendar;
-import java.util.LinkedHashMap;
import java.util.List;
-import java.util.Map;
+/**
+ * Class that contains logic to translate Hive expressions ({@link ExprNodeDesc})
+ * into Calcite expressions ({@link RexNode}).
+ */
public class RexNodeConverter {
- private static class InputCtx {
- private final RelDataType calciteInpDataType;
- private final ImmutableMap hiveNameToPosMap;
- private final RowResolver hiveRR;
- private final int offsetInCalciteSchema;
-
- private InputCtx(RelDataType calciteInpDataType, ImmutableMap hiveNameToPosMap,
- RowResolver hiveRR, int offsetInCalciteSchema) {
- this.calciteInpDataType = calciteInpDataType;
- this.hiveNameToPosMap = hiveNameToPosMap;
- this.hiveRR = hiveRR;
- this.offsetInCalciteSchema = offsetInCalciteSchema;
- }
- };
-
- private final RelOptCluster cluster;
- private final ImmutableList inputCtxs;
- private final boolean flattenExpr;
-
- //outerRR belongs to outer query and is required to resolve correlated references
- private final RowResolver outerRR;
- private final ImmutableMap outerNameToPosMap;
- private int correlatedId;
- private final int maxNodesForInToOrTransformation;
-
- //Constructor used by HiveRexExecutorImpl
- public RexNodeConverter(RelOptCluster cluster) {
- this(cluster, new ArrayList(), false);
- }
-
- //subqueries will need outer query's row resolver
- public RexNodeConverter(RelOptCluster cluster, RelDataType inpDataType,
- ImmutableMap outerNameToPosMap,
- ImmutableMap nameToPosMap, RowResolver hiveRR, RowResolver outerRR,
- int maxNodesForInToOrTransformation, int offset, boolean flattenExpr, int correlatedId) {
- this.cluster = cluster;
- this.inputCtxs = ImmutableList.of(new InputCtx(inpDataType, nameToPosMap, hiveRR, offset));
- this.flattenExpr = flattenExpr;
- this.outerRR = outerRR;
- this.outerNameToPosMap = outerNameToPosMap;
- this.correlatedId = correlatedId;
- this.maxNodesForInToOrTransformation = maxNodesForInToOrTransformation;
- }
+ private final RexBuilder rexBuilder;
+ private final RelDataTypeFactory typeFactory;
- public RexNodeConverter(RelOptCluster cluster, RelDataType inpDataType,
- ImmutableMap nameToPosMap, int offset, boolean flattenExpr) {
- this.cluster = cluster;
- this.inputCtxs = ImmutableList.of(new InputCtx(inpDataType, nameToPosMap, null, offset));
- this.flattenExpr = flattenExpr;
- this.outerRR = null;
- this.outerNameToPosMap = null;
- this.maxNodesForInToOrTransformation = 0;
- }
- public RexNodeConverter(RelOptCluster cluster, List inpCtxLst, boolean flattenExpr) {
- this.cluster = cluster;
- this.inputCtxs = ImmutableList. builder().addAll(inpCtxLst).build();
- this.flattenExpr = flattenExpr;
- this.outerRR = null;
- this.outerNameToPosMap = null;
- this.maxNodesForInToOrTransformation = 0;
+ /**
+ * Constructor used by HiveRexExecutorImpl.
+ */
+ public RexNodeConverter(RexBuilder rexBuilder, RelDataTypeFactory typeFactory) {
+ this.rexBuilder = rexBuilder;
+ this.typeFactory = typeFactory;
}
public RexNode convert(ExprNodeDesc expr) throws SemanticException {
@@ -188,128 +123,19 @@ public RexNode convert(ExprNodeDesc expr) throws SemanticException {
return convert((ExprNodeGenericFuncDesc) expr);
} else if (expr instanceof ExprNodeConstantDesc) {
return convert((ExprNodeConstantDesc) expr);
- } else if (expr instanceof ExprNodeColumnDesc) {
- return convert((ExprNodeColumnDesc) expr);
} else if (expr instanceof ExprNodeFieldDesc) {
return convert((ExprNodeFieldDesc) expr);
- } else if(expr instanceof ExprNodeSubQueryDesc) {
- return convert((ExprNodeSubQueryDesc) expr);
} else {
throw new RuntimeException("Unsupported Expression");
}
- // TODO: handle ExprNodeColumnListDesc
- }
-
- private RexNode getSomeSubquery(final RelNode subqueryRel, final RexNode lhs,
- final SqlQuantifyOperator quantifyOperator) {
- if(quantifyOperator == SqlStdOperatorTable.SOME_EQ) {
- return RexSubQuery.in(subqueryRel, ImmutableList.of(lhs));
- } else if (quantifyOperator == SqlStdOperatorTable.SOME_NE) {
- RexSubQuery subQuery = RexSubQuery.in(subqueryRel, ImmutableList.of(lhs));
- return cluster.getRexBuilder().makeCall(SqlStdOperatorTable.NOT, subQuery);
- } else {
- return RexSubQuery.some(subqueryRel, ImmutableList.of(lhs), quantifyOperator);
- }
- }
-
- private void throwInvalidSubqueryError(final ASTNode comparisonOp) throws SemanticException {
- throw new CalciteSubquerySemanticException(ErrorMsg.INVALID_SUBQUERY_EXPRESSION.getMsg(
- "Invalid operator:" + comparisonOp.toString()));
- }
-
- // <>ANY and =ALL is not supported
- private RexNode convertSubquerySomeAll(final ExprNodeSubQueryDesc subQueryDesc)
- throws SemanticException {
- assert(subQueryDesc.getType() == SubqueryType.SOME
- || subQueryDesc.getType() == SubqueryType.ALL);
-
- RexNode rexNodeLhs = convert(subQueryDesc.getSubQueryLhs());
- ASTNode comparisonOp = subQueryDesc.getComparisonOp();
- SqlQuantifyOperator quantifyOperator = null;
-
- switch (comparisonOp.getType()) {
- case HiveParser.EQUAL:
- if(subQueryDesc.getType() == SubqueryType.ALL) {
- throwInvalidSubqueryError(comparisonOp);
- }
- quantifyOperator = SqlStdOperatorTable.SOME_EQ;
- break;
- case HiveParser.LESSTHAN:
- quantifyOperator = SqlStdOperatorTable.SOME_LT;
- break;
- case HiveParser.LESSTHANOREQUALTO:
- quantifyOperator = SqlStdOperatorTable.SOME_LE;
- break;
- case HiveParser.GREATERTHAN:
- quantifyOperator = SqlStdOperatorTable.SOME_GT;
- break;
- case HiveParser.GREATERTHANOREQUALTO:
- quantifyOperator = SqlStdOperatorTable.SOME_GE;
- break;
- case HiveParser.NOTEQUAL:
- if(subQueryDesc.getType() == SubqueryType.SOME) {
- throwInvalidSubqueryError(comparisonOp);
- }
- quantifyOperator = SqlStdOperatorTable.SOME_NE;
- break;
- default:
- throw new CalciteSubquerySemanticException(ErrorMsg.INVALID_SUBQUERY_EXPRESSION.getMsg(
- "Invalid operator:" + comparisonOp.toString()));
- }
-
- if(subQueryDesc.getType() == SubqueryType.ALL) {
- quantifyOperator = SqlStdOperatorTable.some(quantifyOperator.comparisonKind.negateNullSafe());
- }
- RexNode someQuery = getSomeSubquery(subQueryDesc.getRexSubQuery(), rexNodeLhs,
- quantifyOperator);
- if(subQueryDesc.getType() == SubqueryType.ALL) {
- return cluster.getRexBuilder().makeCall(SqlStdOperatorTable.NOT, someQuery);
- }
- return someQuery;
- }
-
- private RexNode convert(final ExprNodeSubQueryDesc subQueryDesc) throws SemanticException {
- if(subQueryDesc.getType() == SubqueryType.IN) {
- /*
- * Check.5.h :: For In and Not In the SubQuery must implicitly or
- * explicitly only contain one select item.
- */
- if(subQueryDesc.getRexSubQuery().getRowType().getFieldCount() > 1) {
- throw new CalciteSubquerySemanticException(ErrorMsg.INVALID_SUBQUERY_EXPRESSION.getMsg(
- "SubQuery can contain only 1 item in Select List."));
- }
- //create RexNode for LHS
- RexNode rexNodeLhs = convert(subQueryDesc.getSubQueryLhs());
-
- //create RexSubQuery node
- RexNode rexSubQuery = RexSubQuery.in(subQueryDesc.getRexSubQuery(),
- ImmutableList.of(rexNodeLhs));
- return rexSubQuery;
- } else if(subQueryDesc.getType() == SubqueryType.EXISTS) {
- RexNode subQueryNode = RexSubQuery.exists(subQueryDesc.getRexSubQuery());
- return subQueryNode;
- } else if(subQueryDesc.getType() == SubqueryType.SCALAR){
- if(subQueryDesc.getRexSubQuery().getRowType().getFieldCount() > 1) {
- throw new CalciteSubquerySemanticException(ErrorMsg.INVALID_SUBQUERY_EXPRESSION.getMsg(
- "SubQuery can contain only 1 item in Select List."));
- }
- //create RexSubQuery node
- RexNode rexSubQuery = RexSubQuery.scalar(subQueryDesc.getRexSubQuery());
- return rexSubQuery;
- } else if(subQueryDesc.getType() == SubqueryType.SOME
- || subQueryDesc.getType() == SubqueryType.ALL) {
- return convertSubquerySomeAll(subQueryDesc);
- } else {
- throw new CalciteSubquerySemanticException(ErrorMsg.INVALID_SUBQUERY_EXPRESSION.getMsg(
- "Invalid subquery: " + subQueryDesc.getType()));
- }
+ // TODO: Handle ExprNodeColumnDesc, ExprNodeColumnListDesc
}
private RexNode convert(final ExprNodeFieldDesc fieldDesc) throws SemanticException {
RexNode rexNode = convert(fieldDesc.getDesc());
if (rexNode.getType().isStruct()) {
// regular case of accessing nested field in a column
- return cluster.getRexBuilder().makeFieldAccess(rexNode, fieldDesc.getFieldName(), true);
+ return rexBuilder.makeFieldAccess(rexNode, fieldDesc.getFieldName(), true);
} else {
// This may happen for schema-less tables, where columns are dynamically
// supplied by serdes.
@@ -399,32 +225,31 @@ private RexNode convert(ExprNodeGenericFuncDesc func) throws SemanticException {
isAllPrimitive =
isAllPrimitive && tmpExprNode.getTypeInfo().getCategory() == Category.PRIMITIVE;
- argTypeBldr.add(TypeConverter.convert(tmpExprNode.getTypeInfo(), cluster.getTypeFactory()));
+ argTypeBldr.add(TypeConverter.convert(tmpExprNode.getTypeInfo(), typeFactory));
tmpRN = convert(tmpExprNode);
childRexNodeLst.add(tmpRN);
}
// See if this is an explicit cast.
- RexNode expr = null;
- RelDataType retType = null;
- expr = handleExplicitCast(func, childRexNodeLst);
+ RelDataType retType = TypeConverter.convert(func.getTypeInfo(), typeFactory);
+ RexNode expr = handleExplicitCast(func.getGenericUDF(), retType, childRexNodeLst,
+ rexBuilder);
if (expr == null) {
// This is not a cast; process the function.
- retType = TypeConverter.convert(func.getTypeInfo(), cluster.getTypeFactory());
SqlOperator calciteOp = SqlFunctionConverter.getCalciteOperator(func.getFuncText(),
func.getGenericUDF(), argTypeBldr.build(), retType);
if (calciteOp.getKind() == SqlKind.CASE) {
// If it is a case operator, we need to rewrite it
- childRexNodeLst = rewriteCaseChildren(func, childRexNodeLst);
+ childRexNodeLst = rewriteCaseChildren(func.getFuncText(), childRexNodeLst, rexBuilder);
// Adjust branch types by inserting explicit casts if the actual is ambigous
- childRexNodeLst = adjustCaseBranchTypes(childRexNodeLst, retType);
+ childRexNodeLst = adjustCaseBranchTypes(childRexNodeLst, retType, rexBuilder);
} else if (HiveExtractDate.ALL_FUNCTIONS.contains(calciteOp)) {
// If it is a extract operator, we need to rewrite it
- childRexNodeLst = rewriteExtractDateChildren(calciteOp, childRexNodeLst);
+ childRexNodeLst = rewriteExtractDateChildren(calciteOp, childRexNodeLst, rexBuilder);
} else if (HiveFloorDate.ALL_FUNCTIONS.contains(calciteOp)) {
// If it is a floor operator, we need to rewrite it
- childRexNodeLst = rewriteFloorDateChildren(calciteOp, childRexNodeLst);
+ childRexNodeLst = rewriteFloorDateChildren(calciteOp, childRexNodeLst, rexBuilder);
} else if (calciteOp.getKind() == SqlKind.IN && isAllPrimitive) {
if (childRexNodeLst.size() == 2) {
// if it is a single item in an IN clause, transform A IN (B) to A = B
@@ -437,27 +262,19 @@ private RexNode convert(ExprNodeGenericFuncDesc func) throws SemanticException {
// except complex types
// Rewrite to OR is done only if number of operands are less than
// the threshold configured
- boolean rewriteToOr = true;
- if(this.maxNodesForInToOrTransformation != 0) {
- if(childRexNodeLst.size() > this.maxNodesForInToOrTransformation) {
- rewriteToOr = false;
- }
- }
- if(rewriteToOr) {
- childRexNodeLst = rewriteInClauseChildren(calciteOp, childRexNodeLst);
- calciteOp = SqlStdOperatorTable.OR;
- }
+ childRexNodeLst = rewriteInClauseChildren(calciteOp, childRexNodeLst, rexBuilder);
+ calciteOp = SqlStdOperatorTable.OR;
}
} else if (calciteOp.getKind() == SqlKind.COALESCE &&
childRexNodeLst.size() > 1) {
// Rewrite COALESCE as a CASE
// This allows to be further reduced to OR, if possible
calciteOp = SqlStdOperatorTable.CASE;
- childRexNodeLst = rewriteCoalesceChildren(func, childRexNodeLst);
+ childRexNodeLst = rewriteCoalesceChildren(childRexNodeLst, rexBuilder);
// Adjust branch types by inserting explicit casts if the actual is ambigous
- childRexNodeLst = adjustCaseBranchTypes(childRexNodeLst, retType);
+ childRexNodeLst = adjustCaseBranchTypes(childRexNodeLst, retType, rexBuilder);
} else if (calciteOp == HiveToDateSqlOperator.INSTANCE) {
- childRexNodeLst = rewriteToDateChildren(childRexNodeLst);
+ childRexNodeLst = rewriteToDateChildren(childRexNodeLst, rexBuilder);
} else if (calciteOp.getKind() == SqlKind.BETWEEN) {
assert childRexNodeLst.get(0).isAlwaysTrue() || childRexNodeLst.get(0).isAlwaysFalse();
boolean invert = childRexNodeLst.get(0).isAlwaysTrue();
@@ -473,29 +290,29 @@ private RexNode convert(ExprNodeGenericFuncDesc func) throws SemanticException {
RexNode rangeL = childRexNodeLst.get(2);
RexNode rangeH = childRexNodeLst.get(3);
childRexNodeLst.clear();
- childRexNodeLst.add(cluster.getRexBuilder().makeCall(cmpOp, rangeL, op));
- childRexNodeLst.add(cluster.getRexBuilder().makeCall(cmpOp, op, rangeH));
+ childRexNodeLst.add(rexBuilder.makeCall(cmpOp, rangeL, op));
+ childRexNodeLst.add(rexBuilder.makeCall(cmpOp, op, rangeH));
}
- expr = cluster.getRexBuilder().makeCall(retType, calciteOp, childRexNodeLst);
+ expr = rexBuilder.makeCall(retType, calciteOp, childRexNodeLst);
} else {
retType = expr.getType();
}
// TODO: Cast Function in Calcite have a bug where it infer type on cast throws
// an exception
- if (flattenExpr && (expr instanceof RexCall)
+ if (expr instanceof RexCall
&& !(((RexCall) expr).getOperator() instanceof SqlCastFunction)) {
RexCall call = (RexCall) expr;
- expr = cluster.getRexBuilder().makeCall(retType, call.getOperator(),
+ expr = rexBuilder.makeCall(retType, call.getOperator(),
RexUtil.flatten(call.getOperands(), call.getOperator()));
}
return expr;
}
- private boolean castExprUsingUDFBridge(GenericUDF gUDF) {
+ private static boolean castExprUsingUDFBridge(GenericUDF gUDF) {
boolean castExpr = false;
- if (gUDF != null && gUDF instanceof GenericUDFBridge) {
+ if (gUDF instanceof GenericUDFBridge) {
String udfClassName = ((GenericUDFBridge) gUDF).getUdfClassName();
if (udfClassName != null) {
int sp = udfClassName.lastIndexOf('.');
@@ -515,20 +332,17 @@ private boolean castExprUsingUDFBridge(GenericUDF gUDF) {
return castExpr;
}
- private RexNode handleExplicitCast(ExprNodeGenericFuncDesc func, List childRexNodeLst)
- throws CalciteSemanticException {
+ public static RexNode handleExplicitCast(GenericUDF udf, RelDataType returnType, List childRexNodeLst,
+ RexBuilder rexBuilder) {
RexNode castExpr = null;
if (childRexNodeLst != null && childRexNodeLst.size() == 1) {
- GenericUDF udf = func.getGenericUDF();
if ((udf instanceof GenericUDFToChar) || (udf instanceof GenericUDFToVarchar)
|| (udf instanceof GenericUDFToString)
|| (udf instanceof GenericUDFToDecimal) || (udf instanceof GenericUDFToDate)
|| (udf instanceof GenericUDFTimestamp) || (udf instanceof GenericUDFToTimestampLocalTZ)
|| (udf instanceof GenericUDFToBinary) || castExprUsingUDFBridge(udf)) {
- castExpr = cluster.getRexBuilder().makeAbstractCast(
- TypeConverter.convert(func.getTypeInfo(), cluster.getTypeFactory()),
- childRexNodeLst.get(0));
+ castExpr = rexBuilder.makeAbstractCast(returnType, childRexNodeLst.get(0));
}
}
@@ -550,10 +364,10 @@ private RexNode handleExplicitCast(ExprNodeGenericFuncDesc func, List c
* It will be transformed into:
* CASE WHEN =(x + y, 1) THEN 'fee' WHEN =(x + y, 2) THEN 'fie' ELSE null END
*/
- private List rewriteCaseChildren(ExprNodeGenericFuncDesc func, List childRexNodeLst)
- throws SemanticException {
- List newChildRexNodeLst = new ArrayList();
- if (FunctionRegistry.getNormalizedFunctionName(func.getFuncText()).equals("case")) {
+ public static List rewriteCaseChildren(String funcText, List childRexNodeLst,
+ RexBuilder rexBuilder) throws SemanticException {
+ List newChildRexNodeLst = new ArrayList<>();
+ if (FunctionRegistry.getNormalizedFunctionName(funcText).equals("case")) {
RexNode firstPred = childRexNodeLst.get(0);
int length = childRexNodeLst.size() % 2 == 1 ?
childRexNodeLst.size() : childRexNodeLst.size() - 1;
@@ -561,7 +375,7 @@ private RexNode handleExplicitCast(ExprNodeGenericFuncDesc func, List c
if (i % 2 == 1) {
// We rewrite it
newChildRexNodeLst.add(
- cluster.getRexBuilder().makeCall(
+ rexBuilder.makeCall(
SqlStdOperatorTable.EQUALS, firstPred, childRexNodeLst.get(i)));
} else {
newChildRexNodeLst.add(childRexNodeLst.get(i));
@@ -576,7 +390,7 @@ private RexNode handleExplicitCast(ExprNodeGenericFuncDesc func, List c
}
// Calcite always needs the else clause to be defined explicitly
if (newChildRexNodeLst.size() % 2 == 0) {
- newChildRexNodeLst.add(cluster.getRexBuilder().makeNullLiteral(
+ newChildRexNodeLst.add(rexBuilder.makeNullLiteral(
newChildRexNodeLst.get(newChildRexNodeLst.size()-1).getType()));
}
return newChildRexNodeLst;
@@ -588,13 +402,13 @@ private RexNode handleExplicitCast(ExprNodeGenericFuncDesc func, List c
* Calcite is more stricter than hive w.r.t type conversions.
* If a CASE has branches with string/int/boolean branch types; there is no common type.
*/
- private List adjustCaseBranchTypes(List nodes, RelDataType retType) {
+ public static List adjustCaseBranchTypes(List nodes, RelDataType retType, RexBuilder rexBuilder) {
List newNodes = new ArrayList<>();
for (int i = 0; i < nodes.size(); i++) {
RexNode node = nodes.get(i);
if ((i % 2 == 1 || i == nodes.size() - 1)
&& !node.getType().getSqlTypeName().equals(retType.getSqlTypeName())) {
- newNodes.add(cluster.getRexBuilder().makeCast(retType, node));
+ newNodes.add(rexBuilder.makeCast(retType, node));
} else {
newNodes.add(node);
}
@@ -602,33 +416,33 @@ private RexNode handleExplicitCast(ExprNodeGenericFuncDesc func, List