From 9eae5605e00ffe73c673d933c745eb8ba1ae381a Mon Sep 17 00:00:00 2001 From: sunyerui Date: Tue, 26 Jul 2016 19:29:54 +0800 Subject: [PATCH] KYLIN-1921 Support Grouping Functions --- .../org/apache/kylin/query/ITKylinQueryTest.java | 5 + .../test/resources/query/sql_grouping/query00.sql | 25 ++++ .../query/sql_grouping/query00.sql.expected | 1 + .../test/resources/query/sql_grouping/query01.sql | 25 ++++ .../query/sql_grouping/query01.sql.expected | 1 + .../test/resources/query/sql_grouping/query02.sql | 25 ++++ .../query/sql_grouping/query02.sql.expected | 1 + .../query/optrule/AggregateMultipleExpandRule.java | 119 ++++++++++++++++++ .../query/optrule/AggregateProjectReduceRule.java | 96 +++++++++++++++ .../kylin/query/optrule/OLAPAggregateRule.java | 2 +- .../apache/kylin/query/optrule/OLAPUnionRule.java | 48 ++++++++ .../kylin/query/relnode/OLAPAggregateRel.java | 44 +++++-- .../apache/kylin/query/relnode/OLAPProjectRel.java | 6 +- .../apache/kylin/query/relnode/OLAPTableScan.java | 16 +++ .../apache/kylin/query/relnode/OLAPUnionRel.java | 137 +++++++++++++++++++++ 15 files changed, 538 insertions(+), 13 deletions(-) create mode 100644 kylin-it/src/test/resources/query/sql_grouping/query00.sql create mode 100644 kylin-it/src/test/resources/query/sql_grouping/query00.sql.expected create mode 100644 kylin-it/src/test/resources/query/sql_grouping/query01.sql create mode 100644 kylin-it/src/test/resources/query/sql_grouping/query01.sql.expected create mode 100644 kylin-it/src/test/resources/query/sql_grouping/query02.sql create mode 100644 kylin-it/src/test/resources/query/sql_grouping/query02.sql.expected create mode 100644 query/src/main/java/org/apache/kylin/query/optrule/AggregateMultipleExpandRule.java create mode 100644 query/src/main/java/org/apache/kylin/query/optrule/AggregateProjectReduceRule.java create mode 100644 query/src/main/java/org/apache/kylin/query/optrule/OLAPUnionRule.java create mode 100644 query/src/main/java/org/apache/kylin/query/relnode/OLAPUnionRel.java diff --git a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java index b841d11..7541d00 100644 --- a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java +++ b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java @@ -268,6 +268,11 @@ public class ITKylinQueryTest extends KylinTestBase { this.execAndCompQuery(getQueryFolderPrefix() + "src/test/resources/query/sql_raw", null, true); } + @Test + public void testGroupingQuery() throws Exception { + this.batchExecuteQuery(getQueryFolderPrefix() + "src/test/resources/query/sql_grouping"); + } + private void assertLimitWasEnabled() { OLAPContext context = getFirstOLAPContext(); assertTrue(context.storageContext.isLimitEnabled()); diff --git a/kylin-it/src/test/resources/query/sql_grouping/query00.sql b/kylin-it/src/test/resources/query/sql_grouping/query00.sql new file mode 100644 index 0000000..716960d --- /dev/null +++ b/kylin-it/src/test/resources/query/sql_grouping/query00.sql @@ -0,0 +1,25 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +select +(case grouping(cal_dt) when 1 then 'ALL' else cast(cal_dt as varchar(256)) end) as dt, +(case grouping(slr_segment_cd) when 1 then 'ALL' else cast(slr_segment_cd as varchar(256)) end) as cd, +(case grouping(lstg_format_name) when 1 then 'ALL' else lstg_format_name end) as name, +sum(price) as GMV, +count(*) as TRANS_CNT from test_kylin_fact +where cal_dt between '2012-01-01' and '2012-02-01' +group by cube(lstg_format_name, cal_dt, slr_segment_cd) \ No newline at end of file diff --git a/kylin-it/src/test/resources/query/sql_grouping/query00.sql.expected b/kylin-it/src/test/resources/query/sql_grouping/query00.sql.expected new file mode 100644 index 0000000..14b76cb --- /dev/null +++ b/kylin-it/src/test/resources/query/sql_grouping/query00.sql.expected @@ -0,0 +1 @@ +804 diff --git a/kylin-it/src/test/resources/query/sql_grouping/query01.sql b/kylin-it/src/test/resources/query/sql_grouping/query01.sql new file mode 100644 index 0000000..aa824fe --- /dev/null +++ b/kylin-it/src/test/resources/query/sql_grouping/query01.sql @@ -0,0 +1,25 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +select +(case grouping(cal_dt) when 1 then 'ALL' else cast(cal_dt as varchar(256)) end) as dt, +(case grouping(slr_segment_cd) when 1 then 'ALL' else cast(slr_segment_cd as varchar(256)) end) as cd, +(case grouping(lstg_format_name) when 1 then 'ALL' else lstg_format_name end) as name, +sum(price) as GMV, +count(*) as TRANS_CNT from test_kylin_fact +where cal_dt between '2012-01-01' and '2012-02-01' +group by rollup(lstg_format_name, cal_dt, slr_segment_cd) \ No newline at end of file diff --git a/kylin-it/src/test/resources/query/sql_grouping/query01.sql.expected b/kylin-it/src/test/resources/query/sql_grouping/query01.sql.expected new file mode 100644 index 0000000..a9d3bc0 --- /dev/null +++ b/kylin-it/src/test/resources/query/sql_grouping/query01.sql.expected @@ -0,0 +1 @@ +523 \ No newline at end of file diff --git a/kylin-it/src/test/resources/query/sql_grouping/query02.sql b/kylin-it/src/test/resources/query/sql_grouping/query02.sql new file mode 100644 index 0000000..33dec3a --- /dev/null +++ b/kylin-it/src/test/resources/query/sql_grouping/query02.sql @@ -0,0 +1,25 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +select +sum(price) as GMV, +(case grouping(cal_dt) when 1 then 'ALL' else cast(cal_dt as varchar(256)) end) as dt, +(case grouping(slr_segment_cd) when 1 then 'ALL' else cast(slr_segment_cd as varchar(256)) end) as cd, +(case grouping(lstg_format_name) when 1 then 'ALL' else lstg_format_name end) as name, +count(*) as TRANS_CNT from test_kylin_fact +where cal_dt between '2012-01-01' and '2012-02-01' +group by grouping sets((lstg_format_name, cal_dt, slr_segment_cd), (cal_dt, slr_segment_cd), (lstg_format_name, slr_segment_cd)) diff --git a/kylin-it/src/test/resources/query/sql_grouping/query02.sql.expected b/kylin-it/src/test/resources/query/sql_grouping/query02.sql.expected new file mode 100644 index 0000000..5aac265 --- /dev/null +++ b/kylin-it/src/test/resources/query/sql_grouping/query02.sql.expected @@ -0,0 +1 @@ +606 \ No newline at end of file diff --git a/query/src/main/java/org/apache/kylin/query/optrule/AggregateMultipleExpandRule.java b/query/src/main/java/org/apache/kylin/query/optrule/AggregateMultipleExpandRule.java new file mode 100644 index 0000000..f7182f1 --- /dev/null +++ b/query/src/main/java/org/apache/kylin/query/optrule/AggregateMultipleExpandRule.java @@ -0,0 +1,119 @@ +package org.apache.kylin.query.optrule; + +import com.google.common.base.Predicate; +import com.google.common.collect.ImmutableList; +import org.apache.calcite.plan.RelOptRule; +import org.apache.calcite.plan.RelOptRuleCall; +import org.apache.calcite.plan.RelOptRuleOperand; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.core.Aggregate; +import org.apache.calcite.rel.logical.LogicalAggregate; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.RelDataTypeField; +import org.apache.calcite.rex.RexBuilder; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.tools.RelBuilder; +import org.apache.calcite.util.ImmutableBitSet; + +import javax.annotation.Nullable; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; + +/** + * Supoort grouping query. Expand the non-simple aggregate to more than one simple aggregates. + * Add project on expanded simple aggregate to add indicators of origin aggregate. + * All projects on aggregate added into one union, which replace the origin aggregate. + * The new aggregates will be transformed by {@link org.apache.kylin.query.optrule.AggregateProjectReduceRule}, to reduce rolled up dimensions. + * In case to scan other cuboid data without the rolled up dimensions. + * + *

Examples: + *

Origin Aggregate: {@code group by grouping sets ((dim A, dim B, dim C), (dim A, dim C), (dim B, dim C))} + *

Transformed Union: + * {@code select dim A, dim B, dim C, 0, 0, 0 + * union all + * select dim A, null, dim C, 0, 1, 0 + * union all + * select null, dim B, dim C, 1, 0, 0 + * } + */ +public class AggregateMultipleExpandRule extends RelOptRule { + public static final AggregateMultipleExpandRule INSTANCE + = new AggregateMultipleExpandRule(operand(LogicalAggregate.class, null, new Predicate(){ + @Override + public boolean apply(@Nullable Aggregate input) { + return input.getGroupType() != Aggregate.Group.SIMPLE; + } + }, operand(RelNode.class, any())), "AggregateMultipleExpandRule"); + + private AggregateMultipleExpandRule(RelOptRuleOperand operand, String description) { + super(operand, description); + } + + private static List asList(ImmutableBitSet groupSet) { + ArrayList l = new ArrayList<>(1); + l.add(groupSet); + return l; + } + + @Override + public void onMatch(RelOptRuleCall call) { + LogicalAggregate aggr = (LogicalAggregate)call.getRelList().get(0); + RelNode input = aggr.getInput(); + RelBuilder relBuilder = call.builder(); + RexBuilder rexBuilder = aggr.getCluster().getRexBuilder(); + + for (ImmutableBitSet groupSet : aggr.getGroupSets()) { + // push the simple aggregate with one group set + relBuilder.push(aggr.copy(aggr.getTraitSet(), input, false, groupSet, asList(groupSet), aggr.getAggCallList())); + + ImmutableList.Builder rexNodes = new ImmutableList.Builder<>(); + int index = 0; + Iterator groupSetIter = aggr.getGroupSet().iterator(); + Iterator typeIterator = aggr.getRowType().getFieldList().iterator(); + Iterator groupKeyIter = groupSet.iterator(); + int groupKey = groupKeyIter.next(); + + // iterate the group keys, fill with null if the key is rolled up + while (groupSetIter.hasNext()) { + Integer aggrGroupKey = groupSetIter.next(); + RelDataType type = typeIterator.next().getType(); + if (groupKey == aggrGroupKey) { + rexNodes.add(rexBuilder.makeInputRef(type, index++)); + groupKey = groupKeyIter.next(); + continue; + } else { + rexNodes.add(rexBuilder.makeNullLiteral(type.getSqlTypeName())); + } + } + + // fill indicators if need, false when key is present and true if key is rolled up + if (aggr.indicator) { + groupSetIter = aggr.getGroupSet().iterator(); + groupKeyIter = groupSet.iterator(); + groupKey = groupKeyIter.next(); + while (groupSetIter.hasNext()) { + Integer aggrGroupKey = groupSetIter.next(); + RelDataType type = typeIterator.next().getType(); + if (groupKey == aggrGroupKey) { + rexNodes.add(rexBuilder.makeLiteral(false, type, true)); + groupKey = groupKeyIter.next(); + continue; + } else { + rexNodes.add(rexBuilder.makeLiteral(true, type, true)); + } + } + } + + // fill aggr calls input ref + while(typeIterator.hasNext()) { + RelDataType type = typeIterator.next().getType(); + rexNodes.add(rexBuilder.makeInputRef(type, index++)); + } + relBuilder.project(rexNodes.build()); + } + RelNode unionAggr = relBuilder.union(true, aggr.getGroupSets().size()).build(); + + call.transformTo(unionAggr); + } +} diff --git a/query/src/main/java/org/apache/kylin/query/optrule/AggregateProjectReduceRule.java b/query/src/main/java/org/apache/kylin/query/optrule/AggregateProjectReduceRule.java new file mode 100644 index 0000000..7cf685e --- /dev/null +++ b/query/src/main/java/org/apache/kylin/query/optrule/AggregateProjectReduceRule.java @@ -0,0 +1,96 @@ +package org.apache.kylin.query.optrule; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.calcite.plan.RelOptRule; +import org.apache.calcite.plan.RelOptRuleCall; +import org.apache.calcite.plan.RelOptRuleOperand; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.core.Aggregate; +import org.apache.calcite.rel.core.AggregateCall; +import org.apache.calcite.rel.core.RelFactories; +import org.apache.calcite.rel.logical.LogicalAggregate; +import org.apache.calcite.rel.logical.LogicalProject; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.tools.RelBuilder; +import org.apache.calcite.tools.RelBuilderFactory; +import org.apache.calcite.util.ImmutableBitSet; + +import com.google.common.collect.ImmutableList; +import org.apache.calcite.util.Pair; + +/** + * Reduce project under aggregate which has unused input ref. + * The aggregate input ref also need rebuild since project expressions changed. + * Mainly used for the simple aggregates expanded from grouping aggregate by {@link org.apache.kylin.query.optrule.AggregateMultipleExpandRule}. + * With this rule, the rolled up dimensions in aggregate will be reduced, we can use higher layer cuboid data. + */ +public class AggregateProjectReduceRule extends RelOptRule { + public static final AggregateProjectReduceRule INSTANCE + = new AggregateProjectReduceRule(operand(LogicalAggregate.class, null, + Aggregate.IS_SIMPLE, operand(LogicalProject.class, any())), RelFactories.LOGICAL_BUILDER, "AggregateProjectReduceRule"); + + private AggregateProjectReduceRule(RelOptRuleOperand operand, RelBuilderFactory factory, String description) { + super(operand, factory, description); + } + + private void mappingKeys(int key, Pair project, List> projects, Map mapping) { + if (!projects.contains(project)) { + projects.add(project); + } + mapping.put(key, projects.indexOf(project)); + } + + @Override + public void onMatch(RelOptRuleCall call) { + LogicalAggregate aggr = call.rel(0); + LogicalProject project = call.rel(1); + + // generate input ref in group set mapping between old and new project + List> projects = project.getNamedProjects(); + List> newProjects = new ArrayList<>(); + Map mapping = new HashMap<>(); + for (int key : aggr.getGroupSet()) { + mappingKeys(key, projects.get(key), newProjects, mapping); + } + + // create new group set + final ImmutableBitSet newGroupSet = aggr.getGroupSet().permute(mapping); + + // mapping input ref in aggr calls and generate new aggr calls + final ImmutableList.Builder newAggrCalls = ImmutableList.builder(); + for (AggregateCall aggrCall : aggr.getAggCallList()) { + final ImmutableList.Builder newArgs = ImmutableList.builder(); + for (int key : aggrCall.getArgList()) { + mappingKeys(key, projects.get(key), newProjects, mapping); + newArgs.add(mapping.get(key)); + } + final int newFilterArg; + if (aggrCall.filterArg > 0) { + int key = aggrCall.filterArg; + mappingKeys(key, projects.get(key), newProjects, mapping); + newFilterArg = mapping.get(aggrCall.filterArg); + } else { + newFilterArg = -1; + } + + newAggrCalls.add(aggrCall.copy(newArgs.build(), newFilterArg)); + } + + // just return if nothing changed + if (newProjects.equals(project.getNamedProjects())) { + return; + } + + RelBuilder relBuilder = call.builder(); + relBuilder.push(project.getInput()); + relBuilder.project(Pair.left(newProjects), Pair.right(newProjects)); + relBuilder.aggregate(relBuilder.groupKey(newGroupSet, false, null), newAggrCalls.build()); + RelNode rel = relBuilder.build(); + + call.transformTo(rel); + } +} diff --git a/query/src/main/java/org/apache/kylin/query/optrule/OLAPAggregateRule.java b/query/src/main/java/org/apache/kylin/query/optrule/OLAPAggregateRule.java index 2cb73c8..da53152 100644 --- a/query/src/main/java/org/apache/kylin/query/optrule/OLAPAggregateRule.java +++ b/query/src/main/java/org/apache/kylin/query/optrule/OLAPAggregateRule.java @@ -52,7 +52,7 @@ public class OLAPAggregateRule extends ConverterRule { RelTraitSet traitSet = agg.getTraitSet().replace(OLAPRel.CONVENTION); try { - return new OLAPAggregateRel(agg.getCluster(), traitSet, convert(agg.getInput(), traitSet), agg.getGroupSet(), agg.getAggCallList()); + return new OLAPAggregateRel(agg.getCluster(), traitSet, convert(agg.getInput(), traitSet), agg.indicator, agg.getGroupSet(), agg.getGroupSets(), agg.getAggCallList()); } catch (InvalidRelException e) { throw new IllegalStateException("Can't create OLAPAggregateRel!", e); } diff --git a/query/src/main/java/org/apache/kylin/query/optrule/OLAPUnionRule.java b/query/src/main/java/org/apache/kylin/query/optrule/OLAPUnionRule.java new file mode 100644 index 0000000..e3bc7a9 --- /dev/null +++ b/query/src/main/java/org/apache/kylin/query/optrule/OLAPUnionRule.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +package org.apache.kylin.query.optrule; + +import org.apache.calcite.plan.Convention; +import org.apache.calcite.plan.RelTraitSet; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.convert.ConverterRule; +import org.apache.calcite.rel.core.Union; +import org.apache.kylin.query.relnode.OLAPRel; +import org.apache.kylin.query.relnode.OLAPUnionRel; + +import java.util.List; + +/** + */ +public class OLAPUnionRule extends ConverterRule { + + public static final OLAPUnionRule INSTANCE = new OLAPUnionRule(); + + public OLAPUnionRule() { + super(Union.class, Convention.NONE, OLAPRel.CONVENTION, "OLAPUnionRule"); + } + + @Override + public RelNode convert(RelNode rel) { + final Union union = (Union) rel; + final RelTraitSet traitSet = union.getTraitSet().replace(OLAPRel.CONVENTION); + final List inputs = union.getInputs(); + return new OLAPUnionRel(rel.getCluster(), traitSet, convertList(inputs, OLAPRel.CONVENTION), union.all); + } +} diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java index e08c03f..ba74c74 100644 --- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java +++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java @@ -25,6 +25,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import com.google.common.collect.Sets; import org.apache.calcite.adapter.enumerable.EnumerableAggregate; import org.apache.calcite.adapter.enumerable.EnumerableConvention; import org.apache.calcite.adapter.enumerable.EnumerableRel; @@ -105,32 +106,37 @@ public class OLAPAggregateRel extends Aggregate implements OLAPRel { private List groups; private List aggregations; - public OLAPAggregateRel(RelOptCluster cluster, RelTraitSet traits, RelNode child, ImmutableBitSet groupSet, List aggCalls) throws InvalidRelException { - super(cluster, traits, child, false, groupSet, asList(groupSet), aggCalls); + public OLAPAggregateRel(RelOptCluster cluster, RelTraitSet traits, RelNode child, boolean indicator, ImmutableBitSet groupSet, List groupSets, List aggCalls) throws InvalidRelException { + super(cluster, traits, child, indicator, groupSet, groupSets, aggCalls); Preconditions.checkArgument(getConvention() == OLAPRel.CONVENTION); this.afterAggregate = false; this.rewriteAggCalls = aggCalls; this.rowType = getRowType(); } - private static List asList(ImmutableBitSet groupSet) { - ArrayList l = new ArrayList(1); - l.add(groupSet); - return l; - } - @Override public Aggregate copy(RelTraitSet traitSet, RelNode input, boolean indicator, ImmutableBitSet groupSet, List groupSets, List aggCalls) { try { - return new OLAPAggregateRel(getCluster(), traitSet, input, groupSet, aggCalls); + return new OLAPAggregateRel(getCluster(), traitSet, input, indicator, groupSet, groupSets, aggCalls); } catch (InvalidRelException e) { throw new IllegalStateException("Can't create OLAPAggregateRel!", e); } } + /** + * Since the grouping aggregate will be expanded by {@link org.apache.kylin.query.optrule.AggregateMultipleExpandRule}, + * made the cost of grouping aggregate more expensive to use the expanded aggregates + */ @Override public RelOptCost computeSelfCost(RelOptPlanner planner, RelMetadataQuery mq) { - return super.computeSelfCost(planner, mq).multiplyBy(.05); + RelOptCost cost; + if (getGroupType() == Group.SIMPLE) { + cost = super.computeSelfCost(planner, mq).multiplyBy(.05); + } else { + cost = super.computeSelfCost(planner, mq).multiplyBy(.05).plus(planner.getCost(getInput(), mq)) + .multiplyBy(groupSets.size() * 1.5); + } + return cost; } @Override @@ -165,6 +171,22 @@ public class OLAPAggregateRel extends Aggregate implements OLAPRel { List columns = new ArrayList(this.rowType.getFieldCount()); columns.addAll(this.groups); + // Add group column indicators + if (indicator) { + final Set containedNames = Sets.newHashSet(); + for (TblColRef groupCol: groups) { + String base = "i$" + groupCol.getName(); + String name = base; + int i = 0; + while (containedNames.contains(name)) { + name = base + "_" + i++; + } + containedNames.add(name); + TblColRef indicatorCol = TblColRef.newInnerColumn(name, TblColRef.InnerDataTypeEnum.LITERAL); + columns.add(indicatorCol); + } + } + for (int i = 0; i < this.aggregations.size(); i++) { FunctionDesc aggFunc = this.aggregations.get(i); TblColRef aggCol = null; @@ -378,7 +400,7 @@ public class OLAPAggregateRel extends Aggregate implements OLAPRel { public EnumerableRel implementEnumerable(List inputs) { try { return new EnumerableAggregate(getCluster(), getCluster().traitSetOf(EnumerableConvention.INSTANCE), // - sole(inputs), false, this.groupSet, this.groupSets, rewriteAggCalls); + sole(inputs), indicator, this.groupSet, this.groupSets, rewriteAggCalls); } catch (InvalidRelException e) { throw new IllegalStateException("Can't create EnumerableAggregate!", e); } diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPProjectRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPProjectRel.java index 8e454c9..aec4811 100644 --- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPProjectRel.java +++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPProjectRel.java @@ -86,9 +86,13 @@ public class OLAPProjectRel extends Project implements OLAPRel { return rewriteProjects; } + /** + * Since the project under aggregate maybe reduce expressions by {@link org.apache.kylin.query.optrule.AggregateProjectReduceRule}, + * consider the count of expressions into cost, the reduced project will be used. + */ @Override public RelOptCost computeSelfCost(RelOptPlanner planner, RelMetadataQuery mq) { - return super.computeSelfCost(planner, mq).multiplyBy(.05); + return super.computeSelfCost(planner, mq).multiplyBy(.05).multiplyBy(getProjects().size()); } @Override diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java index a3ea30a..bd5b154 100644 --- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java +++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java @@ -44,18 +44,23 @@ import org.apache.calcite.rel.metadata.RelMetadataQuery; import org.apache.calcite.rel.rules.AggregateExpandDistinctAggregatesRule; import org.apache.calcite.rel.rules.AggregateJoinTransposeRule; import org.apache.calcite.rel.rules.AggregateProjectMergeRule; +import org.apache.calcite.rel.rules.AggregateUnionTransposeRule; import org.apache.calcite.rel.rules.FilterJoinRule; import org.apache.calcite.rel.rules.FilterProjectTransposeRule; import org.apache.calcite.rel.rules.JoinCommuteRule; import org.apache.calcite.rel.rules.JoinPushExpressionsRule; import org.apache.calcite.rel.rules.JoinPushThroughJoinRule; +import org.apache.calcite.rel.rules.JoinUnionTransposeRule; import org.apache.calcite.rel.rules.ReduceExpressionsRule; import org.apache.calcite.rel.rules.SortJoinTransposeRule; +import org.apache.calcite.rel.rules.SortUnionTransposeRule; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeFactory; import org.apache.calcite.rel.type.RelDataTypeField; import org.apache.kylin.metadata.model.ColumnDesc; import org.apache.kylin.metadata.model.TblColRef; +import org.apache.kylin.query.optrule.AggregateMultipleExpandRule; +import org.apache.kylin.query.optrule.AggregateProjectReduceRule; import org.apache.kylin.query.optrule.OLAPAggregateRule; import org.apache.kylin.query.optrule.OLAPFilterRule; import org.apache.kylin.query.optrule.OLAPJoinRule; @@ -63,6 +68,7 @@ import org.apache.kylin.query.optrule.OLAPLimitRule; import org.apache.kylin.query.optrule.OLAPProjectRule; import org.apache.kylin.query.optrule.OLAPSortRule; import org.apache.kylin.query.optrule.OLAPToEnumerableConverterRule; +import org.apache.kylin.query.optrule.OLAPUnionRule; import org.apache.kylin.query.schema.OLAPSchema; import org.apache.kylin.query.schema.OLAPTable; @@ -126,6 +132,11 @@ public class OLAPTableScan extends TableScan implements OLAPRel, EnumerableRel { planner.addRule(OLAPJoinRule.INSTANCE); planner.addRule(OLAPLimitRule.INSTANCE); planner.addRule(OLAPSortRule.INSTANCE); + planner.addRule(OLAPUnionRule.INSTANCE); + + // Support translate the grouping aggregate into union of simple aggregates + planner.addRule(AggregateMultipleExpandRule.INSTANCE); + planner.addRule(AggregateProjectReduceRule.INSTANCE); // CalcitePrepareImpl.CONSTANT_REDUCTION_RULES planner.addRule(ReduceExpressionsRule.PROJECT_INSTANCE); @@ -152,9 +163,14 @@ public class OLAPTableScan extends TableScan implements OLAPRel, EnumerableRel { planner.removeRule(FilterProjectTransposeRule.INSTANCE); planner.removeRule(SortJoinTransposeRule.INSTANCE); planner.removeRule(JoinPushExpressionsRule.INSTANCE); + planner.removeRule(SortUnionTransposeRule.INSTANCE); + planner.removeRule(JoinUnionTransposeRule.LEFT_UNION); + planner.removeRule(JoinUnionTransposeRule.RIGHT_UNION); + planner.removeRule(AggregateUnionTransposeRule.INSTANCE); // distinct count will be split into a separated query that is joined with the left query planner.removeRule(AggregateExpandDistinctAggregatesRule.INSTANCE); + // see Dec 26th email @ http://mail-archives.apache.org/mod_mbox/calcite-dev/201412.mbox/browser planner.removeRule(ExpandConversionRule.INSTANCE); } diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPUnionRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPUnionRel.java new file mode 100644 index 0000000..02c3967 --- /dev/null +++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPUnionRel.java @@ -0,0 +1,137 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +package org.apache.kylin.query.relnode; + +import java.util.ArrayList; +import java.util.List; + +import org.apache.calcite.adapter.enumerable.EnumerableConvention; +import org.apache.calcite.adapter.enumerable.EnumerableRel; +import org.apache.calcite.adapter.enumerable.EnumerableUnion; +import org.apache.calcite.plan.RelOptCluster; +import org.apache.calcite.plan.RelOptCost; +import org.apache.calcite.plan.RelOptPlanner; +import org.apache.calcite.plan.RelTrait; +import org.apache.calcite.plan.RelTraitSet; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.RelWriter; +import org.apache.calcite.rel.core.SetOp; +import org.apache.calcite.rel.core.Union; +import org.apache.calcite.rel.metadata.RelMetadataQuery; + +import com.google.common.base.Preconditions; + +/** + */ +public class OLAPUnionRel extends Union implements OLAPRel { + + private final boolean localAll; // avoid same name in parent class + private ColumnRowType columnRowType; + private OLAPContext context; + + public OLAPUnionRel(RelOptCluster cluster, RelTraitSet traitSet, List inputs, boolean all) { + super(cluster, traitSet, inputs, all); + Preconditions.checkArgument(getConvention() == CONVENTION); + for (RelNode child : inputs) { + Preconditions.checkArgument(getConvention() == child.getConvention()); + } + this.localAll = all; + } + + @Override + public SetOp copy(RelTraitSet traitSet, List inputs, boolean all) { + return new OLAPUnionRel(getCluster(), traitSet, inputs, all); + } + + @Override + public RelOptCost computeSelfCost(RelOptPlanner planner, RelMetadataQuery mq) { + return super.computeSelfCost(planner, mq).multiplyBy(.05); + } + + @Override + public RelWriter explainTerms(RelWriter pw) { + return super.explainTerms(pw).itemIf("all", all, true); + } + + @Override + public void implementOLAP(OLAPImplementor implementor) { + for (RelNode child : getInputs()) { + implementor.visitChild(child, this); + } + + this.columnRowType = buildColumnRowType(); + this.context = implementor.getContext(); + } + + private ColumnRowType buildColumnRowType() { + // TODO just hack for now + OLAPRel olapChild = (OLAPRel) getInput(0); + ColumnRowType inputColumnRowType = olapChild.getColumnRowType(); + return inputColumnRowType; + } + + @Override + public void implementRewrite(RewriteImplementor implementor) { + for (RelNode child : getInputs()) { + implementor.visitChild(this, child); + } + + this.rowType = this.deriveRowType(); + this.columnRowType = buildColumnRowType(); + } + + @Override + public EnumerableRel implementEnumerable(List inputs) { + ArrayList relInputs = new ArrayList<>(inputs.size()); + for (EnumerableRel input : inputs) { + if (input instanceof OLAPRel) { + ((OLAPRel) input).replaceTraitSet(EnumerableConvention.INSTANCE); + } + relInputs.add(input); + } + return new EnumerableUnion(getCluster(), traitSet, relInputs, localAll); + } + + @Override + public OLAPContext getContext() { + return context; + } + + @Override + public ColumnRowType getColumnRowType() { + return columnRowType; + } + + @Override + public boolean hasSubQuery() { + for (RelNode child : getInputs()) { + if (((OLAPRel)child).hasSubQuery()) { + return true; + } + } + return false; + } + + @Override + public RelTraitSet replaceTraitSet(RelTrait trait) { + RelTraitSet oldTraitSet = this.traitSet; + this.traitSet = this.traitSet.replace(trait); + return oldTraitSet; + } +} -- 2.3.2 (Apple Git-55)