diff --git a/pom.xml b/pom.xml
index 1682f47d32..4b7c27ea40 100644
--- a/pom.xml
+++ b/pom.xml
@@ -121,7 +121,7 @@
1.10.0
1.7.7
0.8.0.RELEASE
- 1.14.0
+ 1.15.0
4.2.4
4.1.17
4.1.19
@@ -259,6 +259,19 @@
false
+
+ calcite
+ calcite repository
+ https://repository.apache.org/content/repositories/orgapachecalcite-1038
+ default
+
+ true
+ warn
+
+
+ false
+
+
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMaterializedViewsRegistry.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMaterializedViewsRegistry.java
index 51b6ef58fc..03349bbc50 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMaterializedViewsRegistry.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMaterializedViewsRegistry.java
@@ -35,7 +35,6 @@
import org.apache.calcite.adapter.druid.DruidQuery;
import org.apache.calcite.adapter.druid.DruidSchema;
import org.apache.calcite.adapter.druid.DruidTable;
-import org.apache.calcite.adapter.druid.LocalInterval;
import org.apache.calcite.jdbc.JavaTypeFactoryImpl;
import org.apache.calcite.plan.RelOptCluster;
import org.apache.calcite.plan.RelOptMaterialization;
@@ -72,6 +71,7 @@
import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
+import org.joda.time.Interval;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -311,9 +311,8 @@ private static RelNode createTableScan(Table viewTable) {
}
metrics.add(field.getName());
}
- // TODO: Default interval will be an Interval once Calcite 1.15.0 is released.
- // We will need to update the type of this list.
- List intervals = Arrays.asList(DruidTable.DEFAULT_INTERVAL);
+
+ List intervals = Arrays.asList(DruidTable.DEFAULT_INTERVAL);
DruidTable druidTable = new DruidTable(new DruidSchema(address, address, false),
dataSource, RelDataTypeImpl.proto(rowType), metrics, DruidTable.DEFAULT_TIMESTAMP_COLUMN,
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelDistribution.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelDistribution.java
index 653f1c535c..7e5beee573 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelDistribution.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelDistribution.java
@@ -19,6 +19,7 @@
import java.util.List;
+import org.apache.calcite.plan.RelMultipleTrait;
import org.apache.calcite.plan.RelOptPlanner;
import org.apache.calcite.plan.RelTrait;
import org.apache.calcite.plan.RelTraitDef;
@@ -26,8 +27,13 @@
import org.apache.calcite.rel.RelDistributionTraitDef;
import org.apache.calcite.util.mapping.Mappings.TargetMapping;
+import com.google.common.collect.Ordering;
+
public class HiveRelDistribution implements RelDistribution {
+ private static final Ordering> ORDERING =
+ Ordering.natural().lexicographical();
+
List keys;
RelDistribution.Type type;
@@ -77,4 +83,21 @@ public Type getType() {
return type;
}
+ @Override
+ public boolean isTop() {
+ return type == Type.ANY;
+ }
+
+ @Override
+ public int compareTo(RelMultipleTrait o) {
+ final RelDistribution distribution = (RelDistribution) o;
+ if (type == distribution.getType()
+ && (type == Type.HASH_DISTRIBUTED
+ || type == Type.RANGE_DISTRIBUTED)) {
+ return ORDERING.compare(getKeys(), distribution.getKeys());
+ }
+
+ return type.compareTo(distribution.getType());
+ }
+
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveAggregateProjectMergeRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveAggregateProjectMergeRule.java
index b5f17754cb..ec182cd43a 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveAggregateProjectMergeRule.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveAggregateProjectMergeRule.java
@@ -16,28 +16,15 @@
*/
package org.apache.hadoop.hive.ql.optimizer.calcite.rules;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
import org.apache.calcite.plan.RelOptRuleCall;
-import org.apache.calcite.rel.RelNode;
import org.apache.calcite.rel.core.Aggregate;
-import org.apache.calcite.rel.core.Aggregate.Group;
import org.apache.calcite.rel.core.AggregateCall;
import org.apache.calcite.rel.rules.AggregateProjectMergeRule;
-import org.apache.calcite.rex.RexInputRef;
-import org.apache.calcite.rex.RexNode;
-import org.apache.calcite.tools.RelBuilder;
-import org.apache.calcite.util.ImmutableBitSet;
import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelFactories;
import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveAggregate;
import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveGroupingID;
import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.Lists;
-
/**
* Planner rule that recognizes a {@link HiveAggregate}
* on top of a {@link HiveProject} and if possible
@@ -71,95 +58,6 @@ public boolean matches(RelOptRuleCall call) {
return super.matches(call);
}
- public void onMatch(RelOptRuleCall call) {
- final HiveAggregate aggregate = call.rel(0);
- final HiveProject project = call.rel(1);
- RelNode x = apply(call, aggregate, project);
- if (x != null) {
- call.transformTo(x);
- }
- }
-
- public static RelNode apply(RelOptRuleCall call, HiveAggregate aggregate,
- HiveProject project) {
- final List newKeys = Lists.newArrayList();
- final Map map = new HashMap<>();
- for (int key : aggregate.getGroupSet()) {
- final RexNode rex = project.getProjects().get(key);
- if (rex instanceof RexInputRef) {
- final int newKey = ((RexInputRef) rex).getIndex();
- newKeys.add(newKey);
- map.put(key, newKey);
- } else {
- // Cannot handle "GROUP BY expression"
- return null;
- }
- }
-
- final ImmutableBitSet newGroupSet = aggregate.getGroupSet().permute(map);
- ImmutableList newGroupingSets = null;
- if (aggregate.getGroupType() != Group.SIMPLE) {
- newGroupingSets =
- ImmutableBitSet.ORDERING.immutableSortedCopy(
- ImmutableBitSet.permute(aggregate.getGroupSets(), map));
- }
-
- final ImmutableList.Builder aggCalls =
- ImmutableList.builder();
- for (AggregateCall aggregateCall : aggregate.getAggCallList()) {
- final ImmutableList.Builder newArgs = ImmutableList.builder();
- for (int arg : aggregateCall.getArgList()) {
- final RexNode rex = project.getProjects().get(arg);
- if (rex instanceof RexInputRef) {
- newArgs.add(((RexInputRef) rex).getIndex());
- } else {
- // Cannot handle "AGG(expression)"
- return null;
- }
- }
- final int newFilterArg;
- if (aggregateCall.filterArg >= 0) {
- final RexNode rex = project.getProjects().get(aggregateCall.filterArg);
- if (!(rex instanceof RexInputRef)) {
- return null;
- }
- newFilterArg = ((RexInputRef) rex).getIndex();
- } else {
- newFilterArg = -1;
- }
- aggCalls.add(aggregateCall.copy(newArgs.build(), newFilterArg));
- }
-
- final Aggregate newAggregate =
- aggregate.copy(aggregate.getTraitSet(), project.getInput(),
- aggregate.indicator, newGroupSet, newGroupingSets,
- aggCalls.build());
-
- // Add a project if the group set is not in the same order or
- // contains duplicates.
- final RelBuilder relBuilder = call.builder();
- relBuilder.push(newAggregate);
- if (!newKeys.equals(newGroupSet.asList())) {
- final List posList = Lists.newArrayList();
- for (int newKey : newKeys) {
- posList.add(newGroupSet.indexOf(newKey));
- }
- if (aggregate.indicator) {
- for (int newKey : newKeys) {
- posList.add(aggregate.getGroupCount() + newGroupSet.indexOf(newKey));
- }
- }
- for (int i = newAggregate.getGroupCount()
- + newAggregate.getIndicatorCount();
- i < newAggregate.getRowType().getFieldCount(); i++) {
- posList.add(i);
- }
- relBuilder.project(relBuilder.fields(posList));
- }
-
- return relBuilder.build();
- }
-
}
// End HiveAggregateProjectMergeRule.java
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveAggregatePullUpConstantsRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveAggregatePullUpConstantsRule.java
index 370c0eca47..564da1d45c 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveAggregatePullUpConstantsRule.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveAggregatePullUpConstantsRule.java
@@ -20,6 +20,7 @@
import org.apache.calcite.plan.RelOptRuleCall;
import org.apache.calcite.rel.RelNode;
import org.apache.calcite.rel.core.Aggregate;
+import org.apache.calcite.rel.core.Aggregate.Group;
import org.apache.calcite.rel.rules.AggregateProjectPullUpConstantsRule;
import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelFactories;
import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveAggregate;
@@ -38,7 +39,7 @@ public HiveAggregatePullUpConstantsRule() {
public boolean matches(RelOptRuleCall call) {
final Aggregate aggregate = call.rel(0);
// Rule cannot be applied if there are GroupingSets
- if (aggregate.indicator) {
+ if (aggregate.getGroupType() != Group.SIMPLE) {
return false;
}
return super.matches(call);
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveReduceExpressionsRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveReduceExpressionsRule.java
index 9a5d8e2826..d97696105e 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveReduceExpressionsRule.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveReduceExpressionsRule.java
@@ -77,7 +77,7 @@
* {@link org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveJoin}.
*/
public static final ReduceExpressionsRule JOIN_INSTANCE =
- new JoinReduceExpressionsRule(HiveJoin.class, HiveRelFactories.HIVE_BUILDER);
+ new JoinReduceExpressionsRule(HiveJoin.class, false, HiveRelFactories.HIVE_BUILDER);
//~ Constructors -----------------------------------------------------------
@@ -112,7 +112,7 @@ public FilterReduceExpressionsRule(Class extends Filter> filterClass,
final RelMetadataQuery mq = call.getMetadataQuery();
final RelOptPredicateList predicates =
mq.getPulledUpPredicates(filter.getInput());
- if (reduceExpressions(filter, expList, predicates, true)) {
+ if (reduceExpressions(filter, expList, predicates, true, false)) {
assert expList.size() == 1;
newConditionExp = expList.get(0);
reduced = true;
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
index 76c82e2606..7ba70a612c 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
@@ -52,7 +52,6 @@
import org.apache.calcite.adapter.druid.DruidRules;
import org.apache.calcite.adapter.druid.DruidSchema;
import org.apache.calcite.adapter.druid.DruidTable;
-import org.apache.calcite.adapter.druid.LocalInterval;
import org.apache.calcite.config.CalciteConnectionConfig;
import org.apache.calcite.config.CalciteConnectionConfigImpl;
import org.apache.calcite.config.CalciteConnectionProperty;
@@ -259,6 +258,7 @@
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
+import org.joda.time.Interval;
import com.google.common.base.Function;
import com.google.common.collect.ArrayListMultimap;
@@ -2425,9 +2425,8 @@ private RelNode genTableLogicalPlan(String tableAlias, QB qb) throws SemanticExc
}
metrics.add(field.getName());
}
- // TODO: Default interval will be an Interval once Calcite 1.15.0 is released.
- // We will need to update the type of this list.
- List intervals = Arrays.asList(DruidTable.DEFAULT_INTERVAL);
+
+ List intervals = Arrays.asList(DruidTable.DEFAULT_INTERVAL);
DruidTable druidTable = new DruidTable(new DruidSchema(address, address, false),
dataSource, RelDataTypeImpl.proto(rowType), metrics, DruidTable.DEFAULT_TIMESTAMP_COLUMN,