diff --git a/pom.xml b/pom.xml index 1682f47d32..4b7c27ea40 100644 --- a/pom.xml +++ b/pom.xml @@ -121,7 +121,7 @@ 1.10.0 1.7.7 0.8.0.RELEASE - 1.14.0 + 1.15.0 4.2.4 4.1.17 4.1.19 @@ -259,6 +259,19 @@ false + + calcite + calcite repository + https://repository.apache.org/content/repositories/orgapachecalcite-1038 + default + + true + warn + + + false + + diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMaterializedViewsRegistry.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMaterializedViewsRegistry.java index 51b6ef58fc..03349bbc50 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMaterializedViewsRegistry.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMaterializedViewsRegistry.java @@ -35,7 +35,6 @@ import org.apache.calcite.adapter.druid.DruidQuery; import org.apache.calcite.adapter.druid.DruidSchema; import org.apache.calcite.adapter.druid.DruidTable; -import org.apache.calcite.adapter.druid.LocalInterval; import org.apache.calcite.jdbc.JavaTypeFactoryImpl; import org.apache.calcite.plan.RelOptCluster; import org.apache.calcite.plan.RelOptMaterialization; @@ -72,6 +71,7 @@ import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; +import org.joda.time.Interval; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -311,9 +311,8 @@ private static RelNode createTableScan(Table viewTable) { } metrics.add(field.getName()); } - // TODO: Default interval will be an Interval once Calcite 1.15.0 is released. - // We will need to update the type of this list. - List intervals = Arrays.asList(DruidTable.DEFAULT_INTERVAL); + + List intervals = Arrays.asList(DruidTable.DEFAULT_INTERVAL); DruidTable druidTable = new DruidTable(new DruidSchema(address, address, false), dataSource, RelDataTypeImpl.proto(rowType), metrics, DruidTable.DEFAULT_TIMESTAMP_COLUMN, diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelDistribution.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelDistribution.java index 653f1c535c..7e5beee573 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelDistribution.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelDistribution.java @@ -19,6 +19,7 @@ import java.util.List; +import org.apache.calcite.plan.RelMultipleTrait; import org.apache.calcite.plan.RelOptPlanner; import org.apache.calcite.plan.RelTrait; import org.apache.calcite.plan.RelTraitDef; @@ -26,8 +27,13 @@ import org.apache.calcite.rel.RelDistributionTraitDef; import org.apache.calcite.util.mapping.Mappings.TargetMapping; +import com.google.common.collect.Ordering; + public class HiveRelDistribution implements RelDistribution { + private static final Ordering> ORDERING = + Ordering.natural().lexicographical(); + List keys; RelDistribution.Type type; @@ -77,4 +83,21 @@ public Type getType() { return type; } + @Override + public boolean isTop() { + return type == Type.ANY; + } + + @Override + public int compareTo(RelMultipleTrait o) { + final RelDistribution distribution = (RelDistribution) o; + if (type == distribution.getType() + && (type == Type.HASH_DISTRIBUTED + || type == Type.RANGE_DISTRIBUTED)) { + return ORDERING.compare(getKeys(), distribution.getKeys()); + } + + return type.compareTo(distribution.getType()); + } + } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveAggregateProjectMergeRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveAggregateProjectMergeRule.java index b5f17754cb..ec182cd43a 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveAggregateProjectMergeRule.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveAggregateProjectMergeRule.java @@ -16,28 +16,15 @@ */ package org.apache.hadoop.hive.ql.optimizer.calcite.rules; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - import org.apache.calcite.plan.RelOptRuleCall; -import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.core.Aggregate; -import org.apache.calcite.rel.core.Aggregate.Group; import org.apache.calcite.rel.core.AggregateCall; import org.apache.calcite.rel.rules.AggregateProjectMergeRule; -import org.apache.calcite.rex.RexInputRef; -import org.apache.calcite.rex.RexNode; -import org.apache.calcite.tools.RelBuilder; -import org.apache.calcite.util.ImmutableBitSet; import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelFactories; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveAggregate; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveGroupingID; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.Lists; - /** * Planner rule that recognizes a {@link HiveAggregate} * on top of a {@link HiveProject} and if possible @@ -71,95 +58,6 @@ public boolean matches(RelOptRuleCall call) { return super.matches(call); } - public void onMatch(RelOptRuleCall call) { - final HiveAggregate aggregate = call.rel(0); - final HiveProject project = call.rel(1); - RelNode x = apply(call, aggregate, project); - if (x != null) { - call.transformTo(x); - } - } - - public static RelNode apply(RelOptRuleCall call, HiveAggregate aggregate, - HiveProject project) { - final List newKeys = Lists.newArrayList(); - final Map map = new HashMap<>(); - for (int key : aggregate.getGroupSet()) { - final RexNode rex = project.getProjects().get(key); - if (rex instanceof RexInputRef) { - final int newKey = ((RexInputRef) rex).getIndex(); - newKeys.add(newKey); - map.put(key, newKey); - } else { - // Cannot handle "GROUP BY expression" - return null; - } - } - - final ImmutableBitSet newGroupSet = aggregate.getGroupSet().permute(map); - ImmutableList newGroupingSets = null; - if (aggregate.getGroupType() != Group.SIMPLE) { - newGroupingSets = - ImmutableBitSet.ORDERING.immutableSortedCopy( - ImmutableBitSet.permute(aggregate.getGroupSets(), map)); - } - - final ImmutableList.Builder aggCalls = - ImmutableList.builder(); - for (AggregateCall aggregateCall : aggregate.getAggCallList()) { - final ImmutableList.Builder newArgs = ImmutableList.builder(); - for (int arg : aggregateCall.getArgList()) { - final RexNode rex = project.getProjects().get(arg); - if (rex instanceof RexInputRef) { - newArgs.add(((RexInputRef) rex).getIndex()); - } else { - // Cannot handle "AGG(expression)" - return null; - } - } - final int newFilterArg; - if (aggregateCall.filterArg >= 0) { - final RexNode rex = project.getProjects().get(aggregateCall.filterArg); - if (!(rex instanceof RexInputRef)) { - return null; - } - newFilterArg = ((RexInputRef) rex).getIndex(); - } else { - newFilterArg = -1; - } - aggCalls.add(aggregateCall.copy(newArgs.build(), newFilterArg)); - } - - final Aggregate newAggregate = - aggregate.copy(aggregate.getTraitSet(), project.getInput(), - aggregate.indicator, newGroupSet, newGroupingSets, - aggCalls.build()); - - // Add a project if the group set is not in the same order or - // contains duplicates. - final RelBuilder relBuilder = call.builder(); - relBuilder.push(newAggregate); - if (!newKeys.equals(newGroupSet.asList())) { - final List posList = Lists.newArrayList(); - for (int newKey : newKeys) { - posList.add(newGroupSet.indexOf(newKey)); - } - if (aggregate.indicator) { - for (int newKey : newKeys) { - posList.add(aggregate.getGroupCount() + newGroupSet.indexOf(newKey)); - } - } - for (int i = newAggregate.getGroupCount() - + newAggregate.getIndicatorCount(); - i < newAggregate.getRowType().getFieldCount(); i++) { - posList.add(i); - } - relBuilder.project(relBuilder.fields(posList)); - } - - return relBuilder.build(); - } - } // End HiveAggregateProjectMergeRule.java diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveAggregatePullUpConstantsRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveAggregatePullUpConstantsRule.java index 370c0eca47..564da1d45c 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveAggregatePullUpConstantsRule.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveAggregatePullUpConstantsRule.java @@ -20,6 +20,7 @@ import org.apache.calcite.plan.RelOptRuleCall; import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.core.Aggregate; +import org.apache.calcite.rel.core.Aggregate.Group; import org.apache.calcite.rel.rules.AggregateProjectPullUpConstantsRule; import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelFactories; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveAggregate; @@ -38,7 +39,7 @@ public HiveAggregatePullUpConstantsRule() { public boolean matches(RelOptRuleCall call) { final Aggregate aggregate = call.rel(0); // Rule cannot be applied if there are GroupingSets - if (aggregate.indicator) { + if (aggregate.getGroupType() != Group.SIMPLE) { return false; } return super.matches(call); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveReduceExpressionsRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveReduceExpressionsRule.java index 9a5d8e2826..d97696105e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveReduceExpressionsRule.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveReduceExpressionsRule.java @@ -77,7 +77,7 @@ * {@link org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveJoin}. */ public static final ReduceExpressionsRule JOIN_INSTANCE = - new JoinReduceExpressionsRule(HiveJoin.class, HiveRelFactories.HIVE_BUILDER); + new JoinReduceExpressionsRule(HiveJoin.class, false, HiveRelFactories.HIVE_BUILDER); //~ Constructors ----------------------------------------------------------- @@ -112,7 +112,7 @@ public FilterReduceExpressionsRule(Class filterClass, final RelMetadataQuery mq = call.getMetadataQuery(); final RelOptPredicateList predicates = mq.getPulledUpPredicates(filter.getInput()); - if (reduceExpressions(filter, expList, predicates, true)) { + if (reduceExpressions(filter, expList, predicates, true, false)) { assert expList.size() == 1; newConditionExp = expList.get(0); reduced = true; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java index 76c82e2606..7ba70a612c 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java @@ -52,7 +52,6 @@ import org.apache.calcite.adapter.druid.DruidRules; import org.apache.calcite.adapter.druid.DruidSchema; import org.apache.calcite.adapter.druid.DruidTable; -import org.apache.calcite.adapter.druid.LocalInterval; import org.apache.calcite.config.CalciteConnectionConfig; import org.apache.calcite.config.CalciteConnectionConfigImpl; import org.apache.calcite.config.CalciteConnectionProperty; @@ -259,6 +258,7 @@ import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; +import org.joda.time.Interval; import com.google.common.base.Function; import com.google.common.collect.ArrayListMultimap; @@ -2425,9 +2425,8 @@ private RelNode genTableLogicalPlan(String tableAlias, QB qb) throws SemanticExc } metrics.add(field.getName()); } - // TODO: Default interval will be an Interval once Calcite 1.15.0 is released. - // We will need to update the type of this list. - List intervals = Arrays.asList(DruidTable.DEFAULT_INTERVAL); + + List intervals = Arrays.asList(DruidTable.DEFAULT_INTERVAL); DruidTable druidTable = new DruidTable(new DruidSchema(address, address, false), dataSource, RelDataTypeImpl.proto(rowType), metrics, DruidTable.DEFAULT_TIMESTAMP_COLUMN,