diff --git pom.xml pom.xml
index b21d894..e60bb35 100644
--- pom.xml
+++ pom.xml
@@ -100,7 +100,7 @@
3.4
1.7.7
0.8.0.RELEASE
- 1.2.0-incubating
+ 1.3.0-incubating
3.2.6
3.2.10
3.2.9
diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveCalciteUtil.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveCalciteUtil.java
index f4e7c45..199a358 100644
--- ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveCalciteUtil.java
+++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveCalciteUtil.java
@@ -504,7 +504,7 @@ private static JoinLeafPredicateInfo constructJoinLeafPredicateInfo(List sysFieldList,
- RelNode leftRel,
- RelNode rightRel,
- RexNode condition,
- List leftJoinKeys,
- List rightJoinKeys,
- List filterNulls,
- List rangeOp) {
- return splitJoinCondition(
- sysFieldList,
- ImmutableList.of(leftRel, rightRel),
- condition,
- ImmutableList.of(leftJoinKeys, rightJoinKeys),
- filterNulls,
- rangeOp);
- }
-
- /**
- * Splits out the equi-join (and optionally, a single non-equi) components
- * of a join condition, and returns what's left. Projection might be
- * required by the caller to provide join keys that are not direct field
- * references.
- *
- * @param sysFieldList list of system fields
- * @param inputs join inputs
- * @param condition join condition
- * @param joinKeys The join keys from the inputs which are equi-join
- * keys
- * @param filterNulls The join key positions for which null values will not
- * match. null values only match for the "is not distinct
- * from" condition.
- * @param rangeOp if null, only locate equi-joins; otherwise, locate a
- * single non-equi join predicate and return its operator
- * in this list; join keys associated with the non-equi
- * join predicate are at the end of the key lists
- * returned
- * @return What's left, never null
- */
- public static RexNode splitJoinCondition(
- List sysFieldList,
- List inputs,
- RexNode condition,
- List> joinKeys,
- List filterNulls,
- List rangeOp) {
- final List nonEquiList = new ArrayList<>();
-
- splitJoinCondition(
- sysFieldList,
- inputs,
- condition,
- joinKeys,
- filterNulls,
- rangeOp,
- nonEquiList);
-
- // Convert the remainders into a list that are AND'ed together.
- return RexUtil.composeConjunction(
- inputs.get(0).getCluster().getRexBuilder(), nonEquiList, false);
- }
-
- private static void splitJoinCondition(
- List sysFieldList,
- List inputs,
- RexNode condition,
- List> joinKeys,
- List filterNulls,
- List rangeOp,
- List nonEquiList) {
- final int sysFieldCount = sysFieldList.size();
- final RelOptCluster cluster = inputs.get(0).getCluster();
- final RexBuilder rexBuilder = cluster.getRexBuilder();
- final RelDataTypeFactory typeFactory = cluster.getTypeFactory();
-
- int[] firstFieldInputs = new int[inputs.size()];
- int totalFieldCount = 0;
- for (int i = 0; i < inputs.size(); i++) {
- firstFieldInputs[i] = totalFieldCount + sysFieldCount;
- totalFieldCount += sysFieldCount
- + inputs.get(i).getRowType().getFieldCount();
- }
-
- // adjustment array
- int[] adjustments = new int[totalFieldCount];
- for (int i = 0; i < inputs.size(); i++) {
- int limit = i == inputs.size() - 1
- ? totalFieldCount : firstFieldInputs[i + 1];
- for (int j = firstFieldInputs[i]; j < limit; j++) {
- adjustments[j] = -firstFieldInputs[i];
- }
- }
-
- if (condition instanceof RexCall) {
- RexCall call = (RexCall) condition;
- if (call.getOperator() == SqlStdOperatorTable.AND) {
- for (RexNode operand : call.getOperands()) {
- splitJoinCondition(
- sysFieldList,
- inputs,
- operand,
- joinKeys,
- filterNulls,
- rangeOp,
- nonEquiList);
- }
- return;
- }
-
- RexNode leftKey = null;
- RexNode rightKey = null;
- int leftInput = 0;
- int rightInput = 0;
- List leftFields = null;
- List rightFields = null;
- boolean reverse = false;
-
- SqlKind kind = call.getKind();
-
- // Only consider range operators if we haven't already seen one
- if ((kind == SqlKind.EQUALS)
- || (filterNulls != null
- && kind == SqlKind.IS_NOT_DISTINCT_FROM)
- || (rangeOp != null
- && rangeOp.isEmpty()
- && (kind == SqlKind.GREATER_THAN
- || kind == SqlKind.GREATER_THAN_OR_EQUAL
- || kind == SqlKind.LESS_THAN
- || kind == SqlKind.LESS_THAN_OR_EQUAL))) {
- final List operands = call.getOperands();
- RexNode op0 = operands.get(0);
- RexNode op1 = operands.get(1);
-
- final ImmutableBitSet projRefs0 = InputFinder.bits(op0);
- final ImmutableBitSet projRefs1 = InputFinder.bits(op1);
-
- boolean foundBothInputs = false;
- for (int i = 0; i < inputs.size() && !foundBothInputs; i++) {
- final int lowerLimit = firstFieldInputs[i];
- final int upperLimit = i == inputs.size() - 1
- ? totalFieldCount : firstFieldInputs[i + 1];
- if (projRefs0.nextSetBit(lowerLimit) != -1
- && projRefs0.nextSetBit(upperLimit) == -1
- && projRefs0.nextSetBit(0) == projRefs0.nextSetBit(lowerLimit)
- && projRefs0.nextSetBit(lowerLimit) < upperLimit) {
- if (leftKey == null) {
- leftKey = op0;
- leftInput = i;
- leftFields = inputs.get(leftInput).getRowType().getFieldList();
- } else {
- rightKey = op0;
- rightInput = i;
- rightFields = inputs.get(rightInput).getRowType().getFieldList();
- reverse = true;
- foundBothInputs = true;
- }
- } else if (projRefs1.nextSetBit(lowerLimit) != -1
- && projRefs1.nextSetBit(upperLimit) == -1
- && projRefs1.nextSetBit(0) == projRefs1.nextSetBit(lowerLimit)
- && projRefs1.nextSetBit(lowerLimit) < upperLimit) {
- if (leftKey == null) {
- leftKey = op1;
- leftInput = i;
- leftFields = inputs.get(leftInput).getRowType().getFieldList();
- } else {
- rightKey = op1;
- rightInput = i;
- rightFields = inputs.get(rightInput).getRowType().getFieldList();
- foundBothInputs = true;
- }
- }
- }
-
- if ((leftKey != null) && (rightKey != null)) {
- // replace right Key input ref
- rightKey =
- rightKey.accept(
- new RelOptUtil.RexInputConverter(
- rexBuilder,
- rightFields,
- rightFields,
- adjustments));
-
- // left key only needs to be adjusted if there are system
- // fields, but do it for uniformity
- leftKey =
- leftKey.accept(
- new RelOptUtil.RexInputConverter(
- rexBuilder,
- leftFields,
- leftFields,
- adjustments));
-
- RelDataType leftKeyType = leftKey.getType();
- RelDataType rightKeyType = rightKey.getType();
-
- if (leftKeyType != rightKeyType) {
- // perform casting
- RelDataType targetKeyType =
- typeFactory.leastRestrictive(
- ImmutableList.of(leftKeyType, rightKeyType));
-
- if (targetKeyType == null) {
- throw Util.newInternal(
- "Cannot find common type for join keys "
- + leftKey + " (type " + leftKeyType + ") and "
- + rightKey + " (type " + rightKeyType + ")");
- }
-
- if (leftKeyType != targetKeyType) {
- leftKey =
- rexBuilder.makeCast(targetKeyType, leftKey);
- }
-
- if (rightKeyType != targetKeyType) {
- rightKey =
- rexBuilder.makeCast(targetKeyType, rightKey);
- }
- }
- }
- }
-
- if ((rangeOp == null)
- && ((leftKey == null) || (rightKey == null))) {
- // no equality join keys found yet:
- // try transforming the condition to
- // equality "join" conditions, e.g.
- // f(LHS) > 0 ===> ( f(LHS) > 0 ) = TRUE,
- // and make the RHS produce TRUE, but only if we're strictly
- // looking for equi-joins
- final ImmutableBitSet projRefs = InputFinder.bits(condition);
- leftKey = null;
- rightKey = null;
-
- boolean foundInput = false;
- for (int i = 0; i < inputs.size() && !foundInput; i++) {
- final int lowerLimit = firstFieldInputs[i];
- final int upperLimit = i == inputs.size() - 1
- ? totalFieldCount : firstFieldInputs[i + 1];
- if (projRefs.nextSetBit(lowerLimit) < upperLimit) {
- leftInput = i;
- leftFields = inputs.get(leftInput).getRowType().getFieldList();
-
- leftKey = condition.accept(
- new RelOptUtil.RexInputConverter(
- rexBuilder,
- leftFields,
- leftFields,
- adjustments));
-
- rightKey = rexBuilder.makeLiteral(true);
-
- // effectively performing an equality comparison
- kind = SqlKind.EQUALS;
-
- foundInput = true;
- }
- }
- }
-
- if ((leftKey != null) && (rightKey != null)) {
- // found suitable join keys
- // add them to key list, ensuring that if there is a
- // non-equi join predicate, it appears at the end of the
- // key list; also mark the null filtering property
- addJoinKey(
- joinKeys.get(leftInput),
- leftKey,
- (rangeOp != null) && !rangeOp.isEmpty());
- addJoinKey(
- joinKeys.get(rightInput),
- rightKey,
- (rangeOp != null) && !rangeOp.isEmpty());
- if (filterNulls != null
- && kind == SqlKind.EQUALS) {
- // nulls are considered not matching for equality comparison
- // add the position of the most recently inserted key
- filterNulls.add(joinKeys.get(leftInput).size() - 1);
- }
- if (rangeOp != null
- && kind != SqlKind.EQUALS
- && kind != SqlKind.IS_DISTINCT_FROM) {
- if (reverse) {
- kind = reverse(kind);
- }
- rangeOp.add(op(kind, call.getOperator()));
- }
- return;
- } // else fall through and add this condition as nonEqui condition
- }
-
- // The operator is not of RexCall type
- // So we fail. Fall through.
- // Add this condition to the list of non-equi-join conditions.
- nonEquiList.add(condition);
- }
-
- private static SqlKind reverse(SqlKind kind) {
- switch (kind) {
- case GREATER_THAN:
- return SqlKind.LESS_THAN;
- case GREATER_THAN_OR_EQUAL:
- return SqlKind.LESS_THAN_OR_EQUAL;
- case LESS_THAN:
- return SqlKind.GREATER_THAN;
- case LESS_THAN_OR_EQUAL:
- return SqlKind.GREATER_THAN_OR_EQUAL;
- default:
- return kind;
- }
- }
-
- private static SqlOperator op(SqlKind kind, SqlOperator operator) {
- switch (kind) {
- case EQUALS:
- return SqlStdOperatorTable.EQUALS;
- case NOT_EQUALS:
- return SqlStdOperatorTable.NOT_EQUALS;
- case GREATER_THAN:
- return SqlStdOperatorTable.GREATER_THAN;
- case GREATER_THAN_OR_EQUAL:
- return SqlStdOperatorTable.GREATER_THAN_OR_EQUAL;
- case LESS_THAN:
- return SqlStdOperatorTable.LESS_THAN;
- case LESS_THAN_OR_EQUAL:
- return SqlStdOperatorTable.LESS_THAN_OR_EQUAL;
- case IS_DISTINCT_FROM:
- return SqlStdOperatorTable.IS_DISTINCT_FROM;
- case IS_NOT_DISTINCT_FROM:
- return SqlStdOperatorTable.IS_NOT_DISTINCT_FROM;
- default:
- return operator;
- }
- }
-
- private static void addJoinKey(
- List joinKeyList,
- RexNode key,
- boolean preserveLastElementInList) {
- if (!joinKeyList.isEmpty() && preserveLastElementInList) {
- joinKeyList.add(joinKeyList.size() - 1, key);
- } else {
- joinKeyList.add(key);
- }
- }
-
-}
diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveProject.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveProject.java
index cf0c462..4b7887a 100644
--- ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveProject.java
+++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveProject.java
@@ -39,10 +39,10 @@
import org.apache.calcite.util.mapping.Mapping;
import org.apache.calcite.util.mapping.MappingType;
import org.apache.hadoop.hive.ql.optimizer.calcite.CalciteSemanticException;
+import org.apache.hadoop.hive.ql.optimizer.calcite.CalciteSemanticException.UnsupportedFeature;
import org.apache.hadoop.hive.ql.optimizer.calcite.HiveCalciteUtil;
import org.apache.hadoop.hive.ql.optimizer.calcite.TraitsUtil;
-import org.apache.hadoop.hive.ql.optimizer.calcite.CalciteSemanticException.UnsupportedFeature;
-import org.apache.hadoop.hive.ql.optimizer.calcite.cost.HiveCost;
+
import com.google.common.collect.ImmutableList;
public class HiveProject extends Project implements HiveRelNode {
@@ -68,6 +68,7 @@
public HiveProject(RelOptCluster cluster, RelTraitSet traitSet, RelNode child,
List extends RexNode> exps, RelDataType rowType, int flags) {
super(cluster, traitSet, child, exps, rowType, flags);
+ assert traitSet.containsIfApplicable(HiveRelNode.CONVENTION);
virtualCols = ImmutableList.copyOf(HiveCalciteUtil.getVirtualCols(exps));
}
diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveSemiJoin.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveSemiJoin.java
new file mode 100644
index 0000000..dd1691c
--- /dev/null
+++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveSemiJoin.java
@@ -0,0 +1,79 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.optimizer.calcite.reloperators;
+
+import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.plan.RelOptCost;
+import org.apache.calcite.plan.RelOptPlanner;
+import org.apache.calcite.plan.RelTraitSet;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.core.JoinInfo;
+import org.apache.calcite.rel.core.JoinRelType;
+import org.apache.calcite.rel.core.RelFactories.SemiJoinFactory;
+import org.apache.calcite.rel.core.SemiJoin;
+import org.apache.calcite.rel.metadata.RelMetadataQuery;
+import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.util.ImmutableIntList;
+
+public class HiveSemiJoin extends SemiJoin implements HiveRelNode {
+
+ public static final SemiJoinFactory HIVE_SEMIJOIN_FACTORY = new HiveSemiJoinFactoryImpl();
+
+ public HiveSemiJoin(RelOptCluster cluster,
+ RelTraitSet traitSet,
+ RelNode left,
+ RelNode right,
+ RexNode condition,
+ ImmutableIntList leftKeys,
+ ImmutableIntList rightKeys) {
+ super(cluster, traitSet, left, right, condition, leftKeys, rightKeys);
+ }
+
+ @Override
+ public SemiJoin copy(RelTraitSet traitSet, RexNode condition,
+ RelNode left, RelNode right, JoinRelType joinType, boolean semiJoinDone) {
+ final JoinInfo joinInfo = JoinInfo.of(left, right, condition);
+ return new HiveSemiJoin(getCluster(), traitSet, left, right, condition,
+ joinInfo.leftKeys, joinInfo.rightKeys);
+ }
+
+ @Override
+ public void implement(Implementor implementor) {
+ }
+
+ @Override
+ public RelOptCost computeSelfCost(RelOptPlanner planner) {
+ return RelMetadataQuery.getNonCumulativeCost(this);
+ }
+
+ /**
+ * Implementation of {@link SemiJoinFactory} that returns
+ * {@link org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveSemiJoin}
+ * .
+ */
+ private static class HiveSemiJoinFactoryImpl implements SemiJoinFactory {
+ @Override
+ public RelNode createSemiJoin(RelNode left, RelNode right,
+ RexNode condition) {
+ final JoinInfo joinInfo = JoinInfo.of(left, right, condition);
+ final RelOptCluster cluster = left.getCluster();
+ return new HiveSemiJoin(cluster, left.getTraitSet(), left, right, condition,
+ joinInfo.leftKeys, joinInfo.rightKeys);
+ }
+ }
+}
diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/HiveOpConverter.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/HiveOpConverter.java
index 4f19caf..9c21238 100644
--- ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/HiveOpConverter.java
+++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/HiveOpConverter.java
@@ -32,7 +32,6 @@
import org.apache.calcite.rel.RelDistribution.Type;
import org.apache.calcite.rel.RelFieldCollation;
import org.apache.calcite.rel.RelNode;
-import org.apache.calcite.rel.core.SemiJoin;
import org.apache.calcite.rel.rules.MultiJoin;
import org.apache.calcite.rex.RexInputRef;
import org.apache.calcite.rex.RexLiteral;
@@ -63,6 +62,7 @@
import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveFilter;
import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveJoin;
import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject;
+import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveSemiJoin;
import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveSort;
import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveSortExchange;
import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableScan;
@@ -156,8 +156,8 @@ OpAttr dispatch(RelNode rn) throws SemanticException {
return visit((MultiJoin) rn);
} else if (rn instanceof HiveJoin) {
return visit((HiveJoin) rn);
- } else if (rn instanceof SemiJoin) {
- SemiJoin sj = (SemiJoin) rn;
+ } else if (rn instanceof HiveSemiJoin) {
+ HiveSemiJoin sj = (HiveSemiJoin) rn;
HiveJoin hj = HiveJoin.getJoin(sj.getCluster(), sj.getLeft(), sj.getRight(),
sj.getCondition(), sj.getJoinType(), true);
return visit(hj);
diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
index d545bf0..02fc796 100644
--- ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
+++ ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
@@ -59,8 +59,6 @@
import org.apache.calcite.rel.core.Join;
import org.apache.calcite.rel.core.JoinRelType;
import org.apache.calcite.rel.core.Project;
-import org.apache.calcite.rel.core.RelFactories;
-import org.apache.calcite.rel.core.SemiJoin;
import org.apache.calcite.rel.core.Sort;
import org.apache.calcite.rel.metadata.CachingRelMetadataProvider;
import org.apache.calcite.rel.metadata.ChainedRelMetadataProvider;
@@ -121,7 +119,6 @@
import org.apache.hadoop.hive.ql.optimizer.calcite.HiveCalciteUtil;
import org.apache.hadoop.hive.ql.optimizer.calcite.HiveConfigContext;
import org.apache.hadoop.hive.ql.optimizer.calcite.HiveDefaultRelMetadataProvider;
-import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelOptUtil;
import org.apache.hadoop.hive.ql.optimizer.calcite.HiveTypeSystemImpl;
import org.apache.hadoop.hive.ql.optimizer.calcite.RelOptHiveTable;
import org.apache.hadoop.hive.ql.optimizer.calcite.TraitsUtil;
@@ -133,6 +130,7 @@
import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveJoin;
import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject;
import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveRelNode;
+import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveSemiJoin;
import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveSort;
import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableScan;
import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveUnion;
@@ -956,7 +954,7 @@ private RelNode applyPreJoinOrderingTransforms(RelNode basePlan, RelMetadataProv
// 5. Projection Pruning
HiveRelFieldTrimmer fieldTrimmer = new HiveRelFieldTrimmer(null, HiveProject.DEFAULT_PROJECT_FACTORY,
HiveFilter.DEFAULT_FILTER_FACTORY, HiveJoin.HIVE_JOIN_FACTORY,
- RelFactories.DEFAULT_SEMI_JOIN_FACTORY, HiveSort.HIVE_SORT_REL_FACTORY,
+ HiveSemiJoin.HIVE_SEMIJOIN_FACTORY, HiveSort.HIVE_SORT_REL_FACTORY,
HiveAggregate.HIVE_AGGR_REL_FACTORY, HiveUnion.UNION_REL_FACTORY);
basePlan = fieldTrimmer.trim(basePlan);
@@ -1196,7 +1194,7 @@ private RelNode genJoinRelNode(RelNode leftRel, RelNode rightRel, JoinType hiveJ
List leftJoinKeys = new ArrayList();
List rightJoinKeys = new ArrayList();
- RexNode nonEquiConds = HiveRelOptUtil.splitJoinCondition(sysFieldList, leftRel, rightRel,
+ RexNode nonEquiConds = RelOptUtil.splitJoinCondition(sysFieldList, leftRel, rightRel,
calciteJoinCond, leftJoinKeys, rightJoinKeys, null, null);
if (!nonEquiConds.isAlwaysTrue()) {
@@ -1211,7 +1209,7 @@ private RelNode genJoinRelNode(RelNode leftRel, RelNode rightRel, JoinType hiveJ
HiveProject.DEFAULT_PROJECT_FACTORY, inputRels, leftJoinKeys, rightJoinKeys, 0,
leftKeys, rightKeys);
- joinRel = new SemiJoin(cluster, cluster.traitSetOf(HiveRelNode.CONVENTION), inputRels[0],
+ joinRel = new HiveSemiJoin(cluster, cluster.traitSetOf(HiveRelNode.CONVENTION), inputRels[0],
inputRels[1], calciteJoinCond, ImmutableIntList.copyOf(leftKeys),
ImmutableIntList.copyOf(rightKeys));
} else {