From 7f5e42c7b080839f3bb7a1f2a38a800387700021 Mon Sep 17 00:00:00 2001 From: Yang Li Date: Sun, 6 Mar 2016 00:06:05 +0800 Subject: [PATCH] KYLIN-1343 Upgrade to calcite 1.6 (with Edward Zhang) --- .../calcite/adapter/enumerable/EnumerableJoin.java | 145 - .../org/apache/calcite/runtime/SqlFunctions.java | 1317 ---- .../apache/calcite/sql2rel/RelFieldTrimmer.java | 44 + .../apache/calcite/sql2rel/SqlToRelConverter.java | 8150 +++++++++++--------- .../kylin/engine/mr/steps/MergeCuboidJobTest.java | 1 - .../org/apache/kylin/jdbc/KylinConnection.java | 2 +- .../org/apache/kylin/jdbc/KylinJdbcFactory.java | 5 +- .../main/java/org/apache/kylin/jdbc/KylinMeta.java | 46 +- .../apache/kylin/jdbc/KylinPreparedStatement.java | 2 + .../java/org/apache/kylin/jdbc/KylinResultSet.java | 5 +- kylin-it/src/test/resources/logging.properties | 2 +- kylin-it/src/test/resources/query/sql/query92.sql | 30 + .../test/resources/query/sql/query92.sql.disabled | 30 - kylin-it/src/test/resources/query/sql/query93.sql | 30 + .../test/resources/query/sql/query93.sql.disabled | 30 - kylin-it/src/test/resources/query/sql/query94.sql | 30 + .../test/resources/query/sql/query94.sql.disabled | 30 - kylin-it/src/test/resources/query/sql/query95.sql | 30 + .../test/resources/query/sql/query95.sql.disabled | 30 - pom.xml | 2 +- .../apache/kylin/query/optrule/OLAPJoinRule.java | 2 +- .../kylin/query/relnode/OLAPAggregateRel.java | 10 +- .../apache/kylin/query/relnode/OLAPFilterRel.java | 5 +- .../apache/kylin/query/relnode/OLAPJoinRel.java | 22 +- .../apache/kylin/query/relnode/OLAPLimitRel.java | 5 +- .../apache/kylin/query/relnode/OLAPProjectRel.java | 5 +- .../apache/kylin/query/relnode/OLAPSortRel.java | 5 +- .../apache/kylin/query/relnode/OLAPTableScan.java | 9 +- .../query/relnode/OLAPToEnumerableConverter.java | 6 +- .../org/apache/kylin/query/schema/OLAPTable.java | 17 +- 30 files changed, 4836 insertions(+), 5211 deletions(-) delete mode 100644 atopcalcite/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableJoin.java delete mode 100644 atopcalcite/src/main/java/org/apache/calcite/runtime/SqlFunctions.java create mode 100644 atopcalcite/src/main/java/org/apache/calcite/sql2rel/RelFieldTrimmer.java create mode 100644 kylin-it/src/test/resources/query/sql/query92.sql delete mode 100644 kylin-it/src/test/resources/query/sql/query92.sql.disabled create mode 100644 kylin-it/src/test/resources/query/sql/query93.sql delete mode 100644 kylin-it/src/test/resources/query/sql/query93.sql.disabled create mode 100644 kylin-it/src/test/resources/query/sql/query94.sql delete mode 100644 kylin-it/src/test/resources/query/sql/query94.sql.disabled create mode 100644 kylin-it/src/test/resources/query/sql/query95.sql delete mode 100644 kylin-it/src/test/resources/query/sql/query95.sql.disabled diff --git a/atopcalcite/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableJoin.java b/atopcalcite/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableJoin.java deleted file mode 100644 index a3c04f8..0000000 --- a/atopcalcite/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableJoin.java +++ /dev/null @@ -1,145 +0,0 @@ -/* - * OVERRIDE POINT: - * - constructor was private instead of protected - */ - -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to you under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.calcite.adapter.enumerable; - -import java.util.Set; - -import org.apache.calcite.linq4j.tree.BlockBuilder; -import org.apache.calcite.linq4j.tree.Expression; -import org.apache.calcite.linq4j.tree.Expressions; -import org.apache.calcite.plan.RelOptCluster; -import org.apache.calcite.plan.RelOptCost; -import org.apache.calcite.plan.RelOptPlanner; -import org.apache.calcite.plan.RelTraitSet; -import org.apache.calcite.rel.InvalidRelException; -import org.apache.calcite.rel.RelNode; -import org.apache.calcite.rel.RelNodes; -import org.apache.calcite.rel.core.EquiJoin; -import org.apache.calcite.rel.core.JoinInfo; -import org.apache.calcite.rel.core.JoinRelType; -import org.apache.calcite.rel.metadata.RelMetadataQuery; -import org.apache.calcite.rex.RexNode; -import org.apache.calcite.util.BuiltInMethod; -import org.apache.calcite.util.ImmutableIntList; -import org.apache.calcite.util.Util; - -import com.google.common.collect.ImmutableList; - -/** Implementation of {@link org.apache.calcite.rel.core.Join} in - * {@link org.apache.calcite.adapter.enumerable.EnumerableConvention enumerable calling convention}. */ -public class EnumerableJoin extends EquiJoin implements EnumerableRel { - /** Creates an EnumerableJoin. - * - *

Use {@link #create} unless you know what you're doing. */ - public EnumerableJoin(RelOptCluster cluster, RelTraitSet traits, RelNode left, RelNode right, RexNode condition, ImmutableIntList leftKeys, ImmutableIntList rightKeys, JoinRelType joinType, Set variablesStopped) throws InvalidRelException { - super(cluster, traits, left, right, condition, leftKeys, rightKeys, joinType, variablesStopped); - } - - /** Creates an EnumerableJoin. */ - public static EnumerableJoin create(RelNode left, RelNode right, RexNode condition, ImmutableIntList leftKeys, ImmutableIntList rightKeys, JoinRelType joinType, Set variablesStopped) throws InvalidRelException { - final RelOptCluster cluster = left.getCluster(); - final RelTraitSet traitSet = cluster.traitSetOf(EnumerableConvention.INSTANCE); - return new EnumerableJoin(cluster, traitSet, left, right, condition, leftKeys, rightKeys, joinType, variablesStopped); - } - - @Override - public EnumerableJoin copy(RelTraitSet traitSet, RexNode condition, RelNode left, RelNode right, JoinRelType joinType, boolean semiJoinDone) { - final JoinInfo joinInfo = JoinInfo.of(left, right, condition); - assert joinInfo.isEqui(); - try { - return new EnumerableJoin(getCluster(), traitSet, left, right, condition, joinInfo.leftKeys, joinInfo.rightKeys, joinType, variablesStopped); - } catch (InvalidRelException e) { - // Semantic error not possible. Must be a bug. Convert to - // internal error. - throw new AssertionError(e); - } - } - - @Override - public RelOptCost computeSelfCost(RelOptPlanner planner) { - double rowCount = RelMetadataQuery.getRowCount(this); - - // Joins can be flipped, and for many algorithms, both versions are viable - // and have the same cost. To make the results stable between versions of - // the planner, make one of the versions slightly more expensive. - switch (joinType) { - case RIGHT: - rowCount = addEpsilon(rowCount); - break; - default: - if (RelNodes.COMPARATOR.compare(left, right) > 0) { - rowCount = addEpsilon(rowCount); - } - } - - // Cheaper if the smaller number of rows is coming from the LHS. - // Model this by adding L log L to the cost. - final double rightRowCount = right.getRows(); - final double leftRowCount = left.getRows(); - if (Double.isInfinite(leftRowCount)) { - rowCount = leftRowCount; - } else { - rowCount += Util.nLogN(leftRowCount); - } - if (Double.isInfinite(rightRowCount)) { - rowCount = rightRowCount; - } else { - rowCount += rightRowCount; - } - return planner.getCostFactory().makeCost(rowCount, 0, 0); - } - - private double addEpsilon(double d) { - assert d >= 0d; - final double d0 = d; - if (d < 10) { - // For small d, adding 1 would change the value significantly. - d *= 1.001d; - if (d != d0) { - return d; - } - } - // For medium d, add 1. Keeps integral values integral. - ++d; - if (d != d0) { - return d; - } - // For large d, adding 1 might not change the value. Add .1%. - // If d is NaN, this still will probably not change the value. That's OK. - d *= 1.001d; - return d; - } - - public Result implement(EnumerableRelImplementor implementor, Prefer pref) { - BlockBuilder builder = new BlockBuilder(); - final Result leftResult = implementor.visitChild(this, 0, (EnumerableRel) left, pref); - Expression leftExpression = builder.append("left", leftResult.block); - final Result rightResult = implementor.visitChild(this, 1, (EnumerableRel) right, pref); - Expression rightExpression = builder.append("right", rightResult.block); - final PhysType physType = PhysTypeImpl.of(implementor.getTypeFactory(), getRowType(), pref.preferArray()); - final PhysType keyPhysType = leftResult.physType.project(leftKeys, JavaRowFormat.LIST); - return implementor.result(physType, builder.append(Expressions.call(leftExpression, BuiltInMethod.JOIN.method, Expressions.list(rightExpression, leftResult.physType.generateAccessor(leftKeys), rightResult.physType.generateAccessor(rightKeys), EnumUtils.joinSelector(joinType, physType, ImmutableList.of(leftResult.physType, rightResult.physType))).append(Util.first(keyPhysType.comparer(), Expressions.constant(null))).append(Expressions.constant(joinType.generatesNullsOnLeft())).append(Expressions.constant(joinType.generatesNullsOnRight())))).toBlock()); - } - -} - -// End EnumerableJoin.java diff --git a/atopcalcite/src/main/java/org/apache/calcite/runtime/SqlFunctions.java b/atopcalcite/src/main/java/org/apache/calcite/runtime/SqlFunctions.java deleted file mode 100644 index 236601f..0000000 --- a/atopcalcite/src/main/java/org/apache/calcite/runtime/SqlFunctions.java +++ /dev/null @@ -1,1317 +0,0 @@ -/* - * OVERRIDE POINT: - * - divide(BigDecimal,BigDecimal), was `b0.divide(b1)`, now `b0.divide(b1, MathContext.DECIMAL64);` - */ - -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to you under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.calcite.runtime; - -import java.math.BigDecimal; -import java.math.BigInteger; -import java.math.MathContext; -import java.sql.SQLException; -import java.sql.Timestamp; -import java.text.DecimalFormat; -import java.util.Date; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.TimeZone; -import java.util.concurrent.atomic.AtomicLong; -import java.util.regex.Pattern; - -import org.apache.calcite.DataContext; -import org.apache.calcite.avatica.util.ByteString; -import org.apache.calcite.avatica.util.DateTimeUtils; -import org.apache.calcite.linq4j.Enumerable; -import org.apache.calcite.linq4j.Linq4j; -import org.apache.calcite.linq4j.function.Deterministic; -import org.apache.calcite.linq4j.function.Function1; -import org.apache.calcite.linq4j.function.NonDeterministic; -import org.apache.calcite.linq4j.tree.Primitive; - -/** - * Helper methods to implement SQL functions in generated code. - * - *

Not present: and, or, not (builtin operators are better, because they - * use lazy evaluation. Implementations do not check for null values; the - * calling code must do that.

- * - *

Many of the functions do not check for null values. This is intentional. - * If null arguments are possible, the code-generation framework checks for - * nulls before calling the functions.

- */ -@SuppressWarnings("UnnecessaryUnboxing") -@Deterministic -public class SqlFunctions { - private static final DecimalFormat DOUBLE_FORMAT = new DecimalFormat("0.0E0"); - - private static final TimeZone LOCAL_TZ = TimeZone.getDefault(); - - private static final Function1, Enumerable> LIST_AS_ENUMERABLE = new Function1, Enumerable>() { - public Enumerable apply(List list) { - return Linq4j.asEnumerable(list); - } - }; - - /** Holds, for each thread, a map from sequence name to sequence current - * value. - * - *

This is a straw man of an implementation whose main goal is to prove - * that sequences can be parsed, validated and planned. A real application - * will want persistent values for sequences, shared among threads. */ - private static final ThreadLocal> THREAD_SEQUENCES = new ThreadLocal>() { - @Override - protected Map initialValue() { - return new HashMap(); - } - }; - - private SqlFunctions() { - } - - /** SQL SUBSTRING(string FROM ... FOR ...) function. */ - public static String substring(String s, int from, int for_) { - return s.substring(from - 1, Math.min(from - 1 + for_, s.length())); - } - - /** SQL SUBSTRING(string FROM ...) function. */ - public static String substring(String s, int from) { - return s.substring(from - 1); - } - - /** SQL UPPER(string) function. */ - public static String upper(String s) { - return s.toUpperCase(); - } - - /** SQL LOWER(string) function. */ - public static String lower(String s) { - return s.toLowerCase(); - } - - /** SQL INITCAP(string) function. */ - public static String initcap(String s) { - // Assumes Alpha as [A-Za-z0-9] - // white space is treated as everything else. - final int len = s.length(); - boolean start = true; - final StringBuilder newS = new StringBuilder(); - - for (int i = 0; i < len; i++) { - char curCh = s.charAt(i); - final int c = (int) curCh; - if (start) { // curCh is whitespace or first character of word. - if (c > 47 && c < 58) { // 0-9 - start = false; - } else if (c > 64 && c < 91) { // A-Z - start = false; - } else if (c > 96 && c < 123) { // a-z - start = false; - curCh = (char) (c - 32); // Uppercase this character - } - // else {} whitespace - } else { // Inside of a word or white space after end of word. - if (c > 47 && c < 58) { // 0-9 - // noop - } else if (c > 64 && c < 91) { // A-Z - curCh = (char) (c + 32); // Lowercase this character - } else if (c > 96 && c < 123) { // a-z - // noop - } else { // whitespace - start = true; - } - } - newS.append(curCh); - } // for each character in s - return newS.toString(); - } - - /** SQL CHARACTER_LENGTH(string) function. */ - public static int charLength(String s) { - return s.length(); - } - - /** SQL {@code string || string} operator. */ - public static String concat(String s0, String s1) { - return s0 + s1; - } - - /** SQL {@code binary || binary} operator. */ - public static ByteString concat(ByteString s0, ByteString s1) { - return s0.concat(s1); - } - - /** SQL {@code RTRIM} function applied to string. */ - public static String rtrim(String s) { - return trim_(s, false, true, ' '); - } - - /** SQL {@code LTRIM} function. */ - public static String ltrim(String s) { - return trim_(s, true, false, ' '); - } - - /** SQL {@code TRIM(... seek FROM s)} function. */ - public static String trim(boolean leading, boolean trailing, String seek, String s) { - return trim_(s, leading, trailing, seek.charAt(0)); - } - - /** SQL {@code TRIM} function. */ - private static String trim_(String s, boolean left, boolean right, char c) { - int j = s.length(); - if (right) { - for (;;) { - if (j == 0) { - return ""; - } - if (s.charAt(j - 1) != c) { - break; - } - --j; - } - } - int i = 0; - if (left) { - for (;;) { - if (i == j) { - return ""; - } - if (s.charAt(i) != c) { - break; - } - ++i; - } - } - return s.substring(i, j); - } - - /** SQL {@code TRIM} function applied to binary string. */ - public static ByteString trim(ByteString s) { - return trim_(s, true, true); - } - - /** Helper for CAST. */ - public static ByteString rtrim(ByteString s) { - return trim_(s, false, true); - } - - /** SQL {@code TRIM} function applied to binary string. */ - private static ByteString trim_(ByteString s, boolean left, boolean right) { - int j = s.length(); - if (right) { - for (;;) { - if (j == 0) { - return ByteString.EMPTY; - } - if (s.byteAt(j - 1) != 0) { - break; - } - --j; - } - } - int i = 0; - if (left) { - for (;;) { - if (i == j) { - return ByteString.EMPTY; - } - if (s.byteAt(i) != 0) { - break; - } - ++i; - } - } - return s.substring(i, j); - } - - /** SQL {@code OVERLAY} function. */ - public static String overlay(String s, String r, int start) { - if (s == null || r == null) { - return null; - } - return s.substring(0, start - 1) + r + s.substring(start - 1 + r.length()); - } - - /** SQL {@code OVERLAY} function. */ - public static String overlay(String s, String r, int start, int length) { - if (s == null || r == null) { - return null; - } - return s.substring(0, start - 1) + r + s.substring(start - 1 + length); - } - - /** SQL {@code OVERLAY} function applied to binary strings. */ - public static ByteString overlay(ByteString s, ByteString r, int start) { - if (s == null || r == null) { - return null; - } - return s.substring(0, start - 1).concat(r).concat(s.substring(start - 1 + r.length())); - } - - /** SQL {@code OVERLAY} function applied to binary strings. */ - public static ByteString overlay(ByteString s, ByteString r, int start, int length) { - if (s == null || r == null) { - return null; - } - return s.substring(0, start - 1).concat(r).concat(s.substring(start - 1 + length)); - } - - /** SQL {@code LIKE} function. */ - public static boolean like(String s, String pattern) { - final String regex = Like.sqlToRegexLike(pattern, null); - return Pattern.matches(regex, s); - } - - /** SQL {@code LIKE} function with escape. */ - public static boolean like(String s, String pattern, String escape) { - final String regex = Like.sqlToRegexLike(pattern, escape); - return Pattern.matches(regex, s); - } - - /** SQL {@code SIMILAR} function. */ - public static boolean similar(String s, String pattern) { - final String regex = Like.sqlToRegexSimilar(pattern, null); - return Pattern.matches(regex, s); - } - - /** SQL {@code SIMILAR} function with escape. */ - public static boolean similar(String s, String pattern, String escape) { - final String regex = Like.sqlToRegexSimilar(pattern, escape); - return Pattern.matches(regex, s); - } - - // = - - /** SQL = operator applied to Object values (including String; neither - * side may be null). */ - public static boolean eq(Object b0, Object b1) { - return b0.equals(b1); - } - - /** SQL = operator applied to BigDecimal values (neither may be null). */ - public static boolean eq(BigDecimal b0, BigDecimal b1) { - return b0.stripTrailingZeros().equals(b1.stripTrailingZeros()); - } - - // <> - - /** SQL <> operator applied to Object values (including String; - * neither side may be null). */ - public static boolean ne(Object b0, Object b1) { - return !b0.equals(b1); - } - - /** SQL <> operator applied to BigDecimal values. */ - public static boolean ne(BigDecimal b0, BigDecimal b1) { - return b0.compareTo(b1) != 0; - } - - // < - - /** SQL < operator applied to boolean values. */ - public static boolean lt(boolean b0, boolean b1) { - return compare(b0, b1) < 0; - } - - /** SQL < operator applied to String values. */ - public static boolean lt(String b0, String b1) { - return b0.compareTo(b1) < 0; - } - - /** SQL < operator applied to ByteString values. */ - public static boolean lt(ByteString b0, ByteString b1) { - return b0.compareTo(b1) < 0; - } - - /** SQL < operator applied to BigDecimal values. */ - public static boolean lt(BigDecimal b0, BigDecimal b1) { - return b0.compareTo(b1) < 0; - } - - // <= - - /** SQL ≤ operator applied to boolean values. */ - public static boolean le(boolean b0, boolean b1) { - return compare(b0, b1) <= 0; - } - - /** SQL ≤ operator applied to String values. */ - public static boolean le(String b0, String b1) { - return b0.compareTo(b1) <= 0; - } - - /** SQL ≤ operator applied to ByteString values. */ - public static boolean le(ByteString b0, ByteString b1) { - return b0.compareTo(b1) <= 0; - } - - /** SQL ≤ operator applied to BigDecimal values. */ - public static boolean le(BigDecimal b0, BigDecimal b1) { - return b0.compareTo(b1) <= 0; - } - - // > - - /** SQL > operator applied to boolean values. */ - public static boolean gt(boolean b0, boolean b1) { - return compare(b0, b1) > 0; - } - - /** SQL > operator applied to String values. */ - public static boolean gt(String b0, String b1) { - return b0.compareTo(b1) > 0; - } - - /** SQL > operator applied to ByteString values. */ - public static boolean gt(ByteString b0, ByteString b1) { - return b0.compareTo(b1) > 0; - } - - /** SQL > operator applied to BigDecimal values. */ - public static boolean gt(BigDecimal b0, BigDecimal b1) { - return b0.compareTo(b1) > 0; - } - - // >= - - /** SQL ≥ operator applied to boolean values. */ - public static boolean ge(boolean b0, boolean b1) { - return compare(b0, b1) >= 0; - } - - /** SQL ≥ operator applied to String values. */ - public static boolean ge(String b0, String b1) { - return b0.compareTo(b1) >= 0; - } - - /** SQL ≥ operator applied to ByteString values. */ - public static boolean ge(ByteString b0, ByteString b1) { - return b0.compareTo(b1) >= 0; - } - - /** SQL ≥ operator applied to BigDecimal values. */ - public static boolean ge(BigDecimal b0, BigDecimal b1) { - return b0.compareTo(b1) >= 0; - } - - // + - - /** SQL + operator applied to int values. */ - public static int plus(int b0, int b1) { - return b0 + b1; - } - - /** SQL + operator applied to int values; left side may be - * null. */ - public static Integer plus(Integer b0, int b1) { - return b0 == null ? null : (b0 + b1); - } - - /** SQL + operator applied to int values; right side may be - * null. */ - public static Integer plus(int b0, Integer b1) { - return b1 == null ? null : (b0 + b1); - } - - /** SQL + operator applied to nullable int values. */ - public static Integer plus(Integer b0, Integer b1) { - return (b0 == null || b1 == null) ? null : (b0 + b1); - } - - /** SQL + operator applied to nullable long and int values. */ - public static Long plus(Long b0, Integer b1) { - return (b0 == null || b1 == null) ? null : (b0.longValue() + b1.longValue()); - } - - /** SQL + operator applied to nullable int and long values. */ - public static Long plus(Integer b0, Long b1) { - return (b0 == null || b1 == null) ? null : (b0.longValue() + b1.longValue()); - } - - /** SQL + operator applied to BigDecimal values. */ - public static BigDecimal plus(BigDecimal b0, BigDecimal b1) { - return (b0 == null || b1 == null) ? null : b0.add(b1); - } - - // - - - /** SQL - operator applied to int values. */ - public static int minus(int b0, int b1) { - return b0 - b1; - } - - /** SQL - operator applied to int values; left side may be - * null. */ - public static Integer minus(Integer b0, int b1) { - return b0 == null ? null : (b0 - b1); - } - - /** SQL - operator applied to int values; right side may be - * null. */ - public static Integer minus(int b0, Integer b1) { - return b1 == null ? null : (b0 - b1); - } - - /** SQL - operator applied to nullable int values. */ - public static Integer minus(Integer b0, Integer b1) { - return (b0 == null || b1 == null) ? null : (b0 - b1); - } - - /** SQL - operator applied to nullable long and int values. */ - public static Long minus(Long b0, Integer b1) { - return (b0 == null || b1 == null) ? null : (b0.longValue() - b1.longValue()); - } - - /** SQL - operator applied to nullable int and long values. */ - public static Long minus(Integer b0, Long b1) { - return (b0 == null || b1 == null) ? null : (b0.longValue() - b1.longValue()); - } - - /** SQL - operator applied to BigDecimal values. */ - public static BigDecimal minus(BigDecimal b0, BigDecimal b1) { - return (b0 == null || b1 == null) ? null : b0.subtract(b1); - } - - // / - - /** SQL / operator applied to int values. */ - public static int divide(int b0, int b1) { - return b0 / b1; - } - - /** SQL / operator applied to int values; left side may be - * null. */ - public static Integer divide(Integer b0, int b1) { - return b0 == null ? null : (b0 / b1); - } - - /** SQL / operator applied to int values; right side may be - * null. */ - public static Integer divide(int b0, Integer b1) { - return b1 == null ? null : (b0 / b1); - } - - /** SQL / operator applied to nullable int values. */ - public static Integer divide(Integer b0, Integer b1) { - return (b0 == null || b1 == null) ? null : (b0 / b1); - } - - /** SQL / operator applied to nullable long and int values. */ - public static Long divide(Long b0, Integer b1) { - return (b0 == null || b1 == null) ? null : (b0.longValue() / b1.longValue()); - } - - /** SQL / operator applied to nullable int and long values. */ - public static Long divide(Integer b0, Long b1) { - return (b0 == null || b1 == null) ? null : (b0.longValue() / b1.longValue()); - } - - /** SQL / operator applied to BigDecimal values. */ - public static BigDecimal divide(BigDecimal b0, BigDecimal b1) { - return (b0 == null || b1 == null) ? null : b0.divide(b1, MathContext.DECIMAL64); - } - - // * - - /** SQL * operator applied to int values. */ - public static int multiply(int b0, int b1) { - return b0 * b1; - } - - /** SQL * operator applied to int values; left side may be - * null. */ - public static Integer multiply(Integer b0, int b1) { - return b0 == null ? null : (b0 * b1); - } - - /** SQL * operator applied to int values; right side may be - * null. */ - public static Integer multiply(int b0, Integer b1) { - return b1 == null ? null : (b0 * b1); - } - - /** SQL * operator applied to nullable int values. */ - public static Integer multiply(Integer b0, Integer b1) { - return (b0 == null || b1 == null) ? null : (b0 * b1); - } - - /** SQL * operator applied to nullable long and int values. */ - public static Long multiply(Long b0, Integer b1) { - return (b0 == null || b1 == null) ? null : (b0.longValue() * b1.longValue()); - } - - /** SQL * operator applied to nullable int and long values. */ - public static Long multiply(Integer b0, Long b1) { - return (b0 == null || b1 == null) ? null : (b0.longValue() * b1.longValue()); - } - - /** SQL * operator applied to BigDecimal values. */ - public static BigDecimal multiply(BigDecimal b0, BigDecimal b1) { - return (b0 == null || b1 == null) ? null : b0.multiply(b1); - } - - // EXP - - /** SQL EXP operator applied to double values. */ - public static double exp(double b0) { - return Math.exp(b0); - } - - public static double exp(long b0) { - return Math.exp(b0); - } - - // POWER - - /** SQL POWER operator applied to double values. */ - public static double power(double b0, double b1) { - return Math.pow(b0, b1); - } - - public static double power(long b0, long b1) { - return Math.pow(b0, b1); - } - - public static double power(long b0, BigDecimal b1) { - return Math.pow(b0, b1.doubleValue()); - } - - // LN - - /** SQL {@code LN(number)} function applied to double values. */ - public static double ln(double d) { - return Math.log(d); - } - - /** SQL {@code LN(number)} function applied to long values. */ - public static double ln(long b0) { - return Math.log(b0); - } - - /** SQL {@code LN(number)} function applied to BigDecimal values. */ - public static double ln(BigDecimal d) { - return Math.log(d.doubleValue()); - } - - // LOG10 - - /** SQL LOG10(numeric) operator applied to double values. */ - public static double log10(double b0) { - return Math.log10(b0); - } - - /** SQL {@code LOG10(number)} function applied to long values. */ - public static double log10(long b0) { - return Math.log10(b0); - } - - /** SQL {@code LOG10(number)} function applied to BigDecimal values. */ - public static double log10(BigDecimal d) { - return Math.log10(d.doubleValue()); - } - - // MOD - - /** SQL MOD operator applied to byte values. */ - public static byte mod(byte b0, byte b1) { - return (byte) (b0 % b1); - } - - /** SQL MOD operator applied to short values. */ - public static short mod(short b0, short b1) { - return (short) (b0 % b1); - } - - /** SQL MOD operator applied to int values. */ - public static int mod(int b0, int b1) { - return b0 % b1; - } - - /** SQL MOD operator applied to long values. */ - public static long mod(long b0, long b1) { - return b0 % b1; - } - - // temporary - public static BigDecimal mod(BigDecimal b0, int b1) { - return mod(b0, BigDecimal.valueOf(b1)); - } - - // temporary - public static int mod(int b0, BigDecimal b1) { - return mod(b0, b1.intValue()); - } - - public static BigDecimal mod(BigDecimal b0, BigDecimal b1) { - final BigDecimal[] bigDecimals = b0.divideAndRemainder(b1); - return bigDecimals[1]; - } - - // FLOOR - - public static double floor(double b0) { - return Math.floor(b0); - } - - public static float floor(float b0) { - return (float) Math.floor(b0); - } - - public static BigDecimal floor(BigDecimal b0) { - return b0.setScale(0, BigDecimal.ROUND_FLOOR); - } - - /** SQL FLOOR operator applied to byte values. */ - public static byte floor(byte b0, byte b1) { - return (byte) floor((int) b0, (int) b1); - } - - /** SQL FLOOR operator applied to short values. */ - public static short floor(short b0, short b1) { - return (short) floor((int) b0, (int) b1); - } - - /** SQL FLOOR operator applied to int values. */ - public static int floor(int b0, int b1) { - int r = b0 % b1; - if (r < 0) { - r += b1; - } - return b0 - r; - } - - /** SQL FLOOR operator applied to long values. */ - public static long floor(long b0, long b1) { - long r = b0 % b1; - if (r < 0) { - r += b1; - } - return b0 - r; - } - - // temporary - public static BigDecimal floor(BigDecimal b0, int b1) { - return floor(b0, BigDecimal.valueOf(b1)); - } - - // temporary - public static int floor(int b0, BigDecimal b1) { - return floor(b0, b1.intValue()); - } - - public static BigDecimal floor(BigDecimal b0, BigDecimal b1) { - final BigDecimal[] bigDecimals = b0.divideAndRemainder(b1); - BigDecimal r = bigDecimals[1]; - if (r.signum() < 0) { - r = r.add(b1); - } - return b0.subtract(r); - } - - // CEIL - - public static double ceil(double b0) { - return Math.ceil(b0); - } - - public static float ceil(float b0) { - return (float) Math.ceil(b0); - } - - public static BigDecimal ceil(BigDecimal b0) { - return b0.setScale(0, BigDecimal.ROUND_CEILING); - } - - /** SQL CEIL operator applied to byte values. */ - public static byte ceil(byte b0, byte b1) { - return floor((byte) (b0 + b1 - 1), b1); - } - - /** SQL CEIL operator applied to short values. */ - public static short ceil(short b0, short b1) { - return floor((short) (b0 + b1 - 1), b1); - } - - /** SQL CEIL operator applied to int values. */ - public static int ceil(int b0, int b1) { - int r = b0 % b1; - if (r > 0) { - r -= b1; - } - return b0 - r; - } - - /** SQL CEIL operator applied to long values. */ - public static long ceil(long b0, long b1) { - return floor(b0 + b1 - 1, b1); - } - - // temporary - public static BigDecimal ceil(BigDecimal b0, int b1) { - return ceil(b0, BigDecimal.valueOf(b1)); - } - - // temporary - public static int ceil(int b0, BigDecimal b1) { - return ceil(b0, b1.intValue()); - } - - public static BigDecimal ceil(BigDecimal b0, BigDecimal b1) { - final BigDecimal[] bigDecimals = b0.divideAndRemainder(b1); - BigDecimal r = bigDecimals[1]; - if (r.signum() > 0) { - r = r.subtract(b1); - } - return b0.subtract(r); - } - - // ABS - - /** SQL ABS operator applied to byte values. */ - public static byte abs(byte b0) { - return (byte) Math.abs(b0); - } - - /** SQL ABS operator applied to short values. */ - public static short abs(short b0) { - return (short) Math.abs(b0); - } - - /** SQL ABS operator applied to int values. */ - public static int abs(int b0) { - return Math.abs(b0); - } - - /** SQL ABS operator applied to long values. */ - public static long abs(long b0) { - return Math.abs(b0); - } - - /** SQL ABS operator applied to float values. */ - public static float abs(float b0) { - return Math.abs(b0); - } - - /** SQL ABS operator applied to double values. */ - public static double abs(double b0) { - return Math.abs(b0); - } - - /** SQL ABS operator applied to BigDecimal values. */ - public static BigDecimal abs(BigDecimal b0) { - return b0.abs(); - } - - // Helpers - - /** Helper for implementing MIN. Somewhat similar to LEAST operator. */ - public static > T lesser(T b0, T b1) { - return b0 == null || b0.compareTo(b1) > 0 ? b1 : b0; - } - - /** LEAST operator. */ - public static > T least(T b0, T b1) { - return b0 == null || b1 != null && b0.compareTo(b1) > 0 ? b1 : b0; - } - - public static boolean greater(boolean b0, boolean b1) { - return b0 || b1; - } - - public static boolean lesser(boolean b0, boolean b1) { - return b0 && b1; - } - - public static byte greater(byte b0, byte b1) { - return b0 > b1 ? b0 : b1; - } - - public static byte lesser(byte b0, byte b1) { - return b0 > b1 ? b1 : b0; - } - - public static char greater(char b0, char b1) { - return b0 > b1 ? b0 : b1; - } - - public static char lesser(char b0, char b1) { - return b0 > b1 ? b1 : b0; - } - - public static short greater(short b0, short b1) { - return b0 > b1 ? b0 : b1; - } - - public static short lesser(short b0, short b1) { - return b0 > b1 ? b1 : b0; - } - - public static int greater(int b0, int b1) { - return b0 > b1 ? b0 : b1; - } - - public static int lesser(int b0, int b1) { - return b0 > b1 ? b1 : b0; - } - - public static long greater(long b0, long b1) { - return b0 > b1 ? b0 : b1; - } - - public static long lesser(long b0, long b1) { - return b0 > b1 ? b1 : b0; - } - - public static float greater(float b0, float b1) { - return b0 > b1 ? b0 : b1; - } - - public static float lesser(float b0, float b1) { - return b0 > b1 ? b1 : b0; - } - - public static double greater(double b0, double b1) { - return b0 > b1 ? b0 : b1; - } - - public static double lesser(double b0, double b1) { - return b0 > b1 ? b1 : b0; - } - - /** Helper for implementing MAX. Somewhat similar to GREATEST operator. */ - public static > T greater(T b0, T b1) { - return b0 == null || b0.compareTo(b1) < 0 ? b1 : b0; - } - - /** GREATEST operator. */ - public static > T greatest(T b0, T b1) { - return b0 == null || b1 != null && b0.compareTo(b1) < 0 ? b1 : b0; - } - - /** Boolean comparison. */ - public static int compare(boolean x, boolean y) { - return x == y ? 0 : x ? 1 : -1; - } - - /** CAST(FLOAT AS VARCHAR). */ - public static String toString(float x) { - if (x == 0) { - return "0E0"; - } - BigDecimal bigDecimal = new BigDecimal(x, MathContext.DECIMAL32).stripTrailingZeros(); - final String s = bigDecimal.toString(); - return s.replaceAll("0*E", "E").replace("E+", "E"); - } - - /** CAST(DOUBLE AS VARCHAR). */ - public static String toString(double x) { - if (x == 0) { - return "0E0"; - } - BigDecimal bigDecimal = new BigDecimal(x, MathContext.DECIMAL64).stripTrailingZeros(); - final String s = bigDecimal.toString(); - return s.replaceAll("0*E", "E").replace("E+", "E"); - } - - /** CAST(DECIMAL AS VARCHAR). */ - public static String toString(BigDecimal x) { - final String s = x.toString(); - if (s.startsWith("0")) { - // we want ".1" not "0.1" - return s.substring(1); - } else if (s.startsWith("-0")) { - // we want "-.1" not "-0.1" - return "-" + s.substring(2); - } else { - return s; - } - } - - /** CAST(BOOLEAN AS VARCHAR). */ - public static String toString(boolean x) { - // Boolean.toString returns lower case -- no good. - return x ? "TRUE" : "FALSE"; - } - - @NonDeterministic - private static Object cannotConvert(Object o, Class toType) { - throw new RuntimeException("Cannot convert " + o + " to " + toType); - } - - /** CAST(VARCHAR AS BOOLEAN). */ - public static boolean toBoolean(String s) { - s = trim_(s, true, true, ' '); - if (s.equalsIgnoreCase("TRUE")) { - return true; - } else if (s.equalsIgnoreCase("FALSE")) { - return false; - } else { - throw new RuntimeException("Invalid character for cast"); - } - } - - public static boolean toBoolean(Number number) { - return !number.equals(0); - } - - public static boolean toBoolean(Object o) { - return o instanceof Boolean ? (Boolean) o : o instanceof Number ? toBoolean((Number) o) : o instanceof String ? toBoolean((String) o) : (Boolean) cannotConvert(o, boolean.class); - } - - // Don't need parseByte etc. - Byte.parseByte is sufficient. - - public static byte toByte(Object o) { - return o instanceof Byte ? (Byte) o : o instanceof Number ? toByte((Number) o) : Byte.parseByte(o.toString()); - } - - public static byte toByte(Number number) { - return number.byteValue(); - } - - public static char toChar(String s) { - return s.charAt(0); - } - - public static Character toCharBoxed(String s) { - return s.charAt(0); - } - - public static short toShort(String s) { - return Short.parseShort(s.trim()); - } - - public static short toShort(Number number) { - return number.shortValue(); - } - - public static short toShort(Object o) { - return o instanceof Short ? (Short) o : o instanceof Number ? toShort((Number) o) : o instanceof String ? toShort((String) o) : (Short) cannotConvert(o, short.class); - } - - public static int toInt(java.util.Date v) { - return toInt(v, LOCAL_TZ); - } - - public static int toInt(java.util.Date v, TimeZone timeZone) { - return (int) (toLong(v, timeZone) / DateTimeUtils.MILLIS_PER_DAY); - } - - public static Integer toIntOptional(java.util.Date v) { - return v == null ? null : toInt(v); - } - - public static Integer toIntOptional(java.util.Date v, TimeZone timeZone) { - return v == null ? null : toInt(v, timeZone); - } - - public static long toLong(Date v) { - return toLong(v, LOCAL_TZ); - } - - public static int toInt(java.sql.Time v) { - return (int) (toLong(v) % DateTimeUtils.MILLIS_PER_DAY); - } - - public static Integer toIntOptional(java.sql.Time v) { - return v == null ? null : toInt(v); - } - - public static int toInt(String s) { - return Integer.parseInt(s.trim()); - } - - public static int toInt(Number number) { - return number.intValue(); - } - - public static int toInt(Object o) { - return o instanceof Integer ? (Integer) o : o instanceof Number ? toInt((Number) o) : o instanceof String ? toInt((String) o) : (Integer) cannotConvert(o, int.class); - } - - public static long toLong(Timestamp v) { - return toLong(v, LOCAL_TZ); - } - - // mainly intended for java.sql.Timestamp but works for other dates also - public static long toLong(java.util.Date v, TimeZone timeZone) { - final long time = v.getTime(); - return time + timeZone.getOffset(time); - } - - // mainly intended for java.sql.Timestamp but works for other dates also - public static Long toLongOptional(java.util.Date v) { - return v == null ? null : toLong(v, LOCAL_TZ); - } - - public static Long toLongOptional(Timestamp v, TimeZone timeZone) { - if (v == null) { - return null; - } - return toLong(v, LOCAL_TZ); - } - - public static long toLong(String s) { - if (s.startsWith("199") && s.contains(":")) { - return Timestamp.valueOf(s).getTime(); - } - return Long.parseLong(s.trim()); - } - - public static long toLong(Number number) { - return number.longValue(); - } - - public static long toLong(Object o) { - return o instanceof Long ? (Long) o : o instanceof Number ? toLong((Number) o) : o instanceof String ? toLong((String) o) : (Long) cannotConvert(o, long.class); - } - - public static float toFloat(String s) { - return Float.parseFloat(s.trim()); - } - - public static float toFloat(Number number) { - return number.floatValue(); - } - - public static float toFloat(Object o) { - return o instanceof Float ? (Float) o : o instanceof Number ? toFloat((Number) o) : o instanceof String ? toFloat((String) o) : (Float) cannotConvert(o, float.class); - } - - public static double toDouble(String s) { - return Double.parseDouble(s.trim()); - } - - public static double toDouble(Number number) { - return number.doubleValue(); - } - - public static double toDouble(Object o) { - return o instanceof Double ? (Double) o : o instanceof Number ? toDouble((Number) o) : o instanceof String ? toDouble((String) o) : (Double) cannotConvert(o, double.class); - } - - public static BigDecimal toBigDecimal(String s) { - return new BigDecimal(s.trim()); - } - - public static BigDecimal toBigDecimal(Number number) { - // There are some values of "long" that cannot be represented as "double". - // Not so "int". If it isn't a long, go straight to double. - return number instanceof BigDecimal ? (BigDecimal) number : number instanceof BigInteger ? new BigDecimal((BigInteger) number) : number instanceof Long ? new BigDecimal(number.longValue()) : new BigDecimal(number.doubleValue()); - } - - public static BigDecimal toBigDecimal(Object o) { - return o instanceof Number ? toBigDecimal((Number) o) : toBigDecimal(o.toString()); - } - - // Don't need shortValueOf etc. - Short.valueOf is sufficient. - - /** Helper for CAST(... AS VARCHAR(maxLength)). */ - public static String truncate(String s, int maxLength) { - return s == null ? null : s.length() > maxLength ? s.substring(0, maxLength) : s; - } - - /** Helper for CAST(... AS VARBINARY(maxLength)). */ - public static ByteString truncate(ByteString s, int maxLength) { - return s == null ? null : s.length() > maxLength ? s.substring(0, maxLength) : s; - } - - /** SQL {@code POSITION(seek IN string)} function. */ - public static int position(String seek, String s) { - return s.indexOf(seek) + 1; - } - - /** SQL {@code POSITION(seek IN string)} function. */ - public static int position(ByteString seek, ByteString s) { - return s.indexOf(seek) + 1; - } - - /** Helper for rounding. Truncate(12345, 1000) returns 12000. */ - public static long round(long v, long x) { - return truncate(v + x / 2, x); - } - - /** Helper for rounding. Truncate(12345, 1000) returns 12000. */ - public static long truncate(long v, long x) { - long remainder = v % x; - if (remainder < 0) { - remainder += x; - } - return v - remainder; - } - - /** Helper for rounding. Truncate(12345, 1000) returns 12000. */ - public static int round(int v, int x) { - return truncate(v + x / 2, x); - } - - /** Helper for rounding. Truncate(12345, 1000) returns 12000. */ - public static int truncate(int v, int x) { - int remainder = v % x; - if (remainder < 0) { - remainder += x; - } - return v - remainder; - } - - /** SQL {@code CURRENT_TIMESTAMP} function. */ - @NonDeterministic - public static long currentTimestamp(DataContext root) { - // Cast required for JDK 1.6. - return (Long) DataContext.Variable.CURRENT_TIMESTAMP.get(root); - } - - /** SQL {@code CURRENT_TIME} function. */ - @NonDeterministic - public static int currentTime(DataContext root) { - int time = (int) (currentTimestamp(root) % DateTimeUtils.MILLIS_PER_DAY); - if (time < 0) { - time += DateTimeUtils.MILLIS_PER_DAY; - } - return time; - } - - /** SQL {@code CURRENT_DATE} function. */ - @NonDeterministic - public static int currentDate(DataContext root) { - final long timestamp = currentTimestamp(root); - int date = (int) (timestamp / DateTimeUtils.MILLIS_PER_DAY); - final int time = (int) (timestamp % DateTimeUtils.MILLIS_PER_DAY); - if (time < 0) { - --date; - } - return date; - } - - /** SQL {@code LOCAL_TIMESTAMP} function. */ - @NonDeterministic - public static long localTimestamp(DataContext root) { - // Cast required for JDK 1.6. - return (Long) DataContext.Variable.LOCAL_TIMESTAMP.get(root); - } - - /** SQL {@code LOCAL_TIME} function. */ - @NonDeterministic - public static int localTime(DataContext root) { - return (int) (localTimestamp(root) % DateTimeUtils.MILLIS_PER_DAY); - } - - /** Helper for "array element reference". Caller has already ensured that - * array and index are not null. Index is 1-based, per SQL. */ - public static Object arrayItem(List list, int item) { - if (item < 1 || item > list.size()) { - return null; - } - return list.get(item - 1); - } - - /** Helper for "map element reference". Caller has already ensured that - * array and index are not null. Index is 1-based, per SQL. */ - public static Object mapItem(Map map, Object item) { - return map.get(item); - } - - /** Implements the {@code [ ... ]} operator on an object whose type is not - * known until runtime. - */ - public static Object item(Object object, Object index) { - if (object instanceof Map) { - return ((Map) object).get(index); - } - if (object instanceof List && index instanceof Number) { - List list = (List) object; - return list.get(((Number) index).intValue()); - } - return null; - } - - /** NULL → FALSE, FALSE → FALSE, TRUE → TRUE. */ - public static boolean isTrue(Boolean b) { - return b != null && b; - } - - /** NULL → TRUE, FALSE → FALSE, TRUE → TRUE. */ - public static boolean isNotFalse(Boolean b) { - return b == null || b; - } - - /** NULL → NULL, FALSE → TRUE, TRUE → FALSE. */ - public static Boolean not(Boolean b) { - return (b == null) ? null : !b; - } - - /** Converts a JDBC array to a list. */ - public static List arrayToList(final java.sql.Array a) { - if (a == null) { - return null; - } - try { - return Primitive.asList(a.getArray()); - } catch (SQLException e) { - throw new RuntimeException(e); - } - } - - /** Support the {@code CURRENT VALUE OF sequence} operator. */ - @NonDeterministic - public static long sequenceCurrentValue(String key) { - return getAtomicLong(key).get(); - } - - /** Support the {@code NEXT VALUE OF sequence} operator. */ - @NonDeterministic - public static long sequenceNextValue(String key) { - return getAtomicLong(key).incrementAndGet(); - } - - private static AtomicLong getAtomicLong(String key) { - final Map map = THREAD_SEQUENCES.get(); - AtomicLong atomic = map.get(key); - if (atomic == null) { - atomic = new AtomicLong(); - map.put(key, atomic); - } - return atomic; - } - - /** Support the SLICE function. */ - public static List slice(List list) { - return list; - } - - /** Support the ELEMENT function. */ - public static Object element(List list) { - switch (list.size()) { - case 0: - return null; - case 1: - return list.get(0); - default: - throw new RuntimeException("more than one value"); - } - } - - /** Returns a lambda that converts a list to an enumerable. */ - public static Function1, Enumerable> listToEnumerable() { - //noinspection unchecked - return (Function1, Enumerable>) (Function1) LIST_AS_ENUMERABLE; - } - -} - -// End SqlFunctions.java diff --git a/atopcalcite/src/main/java/org/apache/calcite/sql2rel/RelFieldTrimmer.java b/atopcalcite/src/main/java/org/apache/calcite/sql2rel/RelFieldTrimmer.java new file mode 100644 index 0000000..f88157c --- /dev/null +++ b/atopcalcite/src/main/java/org/apache/calcite/sql2rel/RelFieldTrimmer.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.calcite.sql2rel; + +import org.apache.calcite.plan.RelOptCluster; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.core.RelFactories; +import org.apache.calcite.sql.validate.SqlValidator; +import org.apache.calcite.tools.RelBuilder; + +/* + * OVERRIDE POINT: + * - disable the whole RelFieldTrimmer + */ + +public class RelFieldTrimmer { + + public RelFieldTrimmer(SqlValidator validator, RelBuilder relBuilder) { + } + + public RelFieldTrimmer(SqlValidator validator, RelOptCluster cluster, RelFactories.ProjectFactory projectFactory, RelFactories.FilterFactory filterFactory, RelFactories.JoinFactory joinFactory, RelFactories.SemiJoinFactory semiJoinFactory, RelFactories.SortFactory sortFactory, RelFactories.AggregateFactory aggregateFactory, RelFactories.SetOpFactory setOpFactory) { + } + + public RelNode trim(RelNode rootRel) { + return rootRel; + } + +} diff --git a/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java b/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java index c184f0b..aed7c27 100644 --- a/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java +++ b/atopcalcite/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java @@ -18,6 +18,7 @@ package org.apache.calcite.sql2rel; +import org.apache.calcite.avatica.util.Spaces; import org.apache.calcite.linq4j.Ord; import org.apache.calcite.plan.Convention; import org.apache.calcite.plan.RelOptCluster; @@ -25,26 +26,32 @@ import org.apache.calcite.plan.RelOptPlanner; import org.apache.calcite.plan.RelOptSamplingParameters; import org.apache.calcite.plan.RelOptTable; import org.apache.calcite.plan.RelOptUtil; +import org.apache.calcite.plan.RelTraitSet; import org.apache.calcite.prepare.Prepare; import org.apache.calcite.prepare.RelOptTableImpl; import org.apache.calcite.rel.RelCollation; +import org.apache.calcite.rel.RelCollationTraitDef; import org.apache.calcite.rel.RelCollations; import org.apache.calcite.rel.RelFieldCollation; import org.apache.calcite.rel.RelNode; -import org.apache.calcite.rel.RelShuttle; +import org.apache.calcite.rel.RelRoot; +import org.apache.calcite.rel.SingleRel; import org.apache.calcite.rel.core.Aggregate; import org.apache.calcite.rel.core.AggregateCall; import org.apache.calcite.rel.core.Collect; import org.apache.calcite.rel.core.CorrelationId; +import org.apache.calcite.rel.core.Filter; import org.apache.calcite.rel.core.Join; import org.apache.calcite.rel.core.JoinInfo; import org.apache.calcite.rel.core.JoinRelType; import org.apache.calcite.rel.core.Project; import org.apache.calcite.rel.core.RelFactories; import org.apache.calcite.rel.core.Sample; +import org.apache.calcite.rel.core.Sort; import org.apache.calcite.rel.core.Uncollect; import org.apache.calcite.rel.logical.LogicalAggregate; import org.apache.calcite.rel.logical.LogicalCorrelate; +import org.apache.calcite.rel.logical.LogicalFilter; import org.apache.calcite.rel.logical.LogicalIntersect; import org.apache.calcite.rel.logical.LogicalJoin; import org.apache.calcite.rel.logical.LogicalMinus; @@ -56,6 +63,7 @@ import org.apache.calcite.rel.logical.LogicalTableScan; import org.apache.calcite.rel.logical.LogicalUnion; import org.apache.calcite.rel.logical.LogicalValues; import org.apache.calcite.rel.metadata.RelColumnMapping; +import org.apache.calcite.rel.stream.Delta; import org.apache.calcite.rel.stream.LogicalDelta; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeFactory; @@ -72,8 +80,8 @@ import org.apache.calcite.rex.RexLiteral; import org.apache.calcite.rex.RexNode; import org.apache.calcite.rex.RexRangeRef; import org.apache.calcite.rex.RexShuttle; +import org.apache.calcite.rex.RexSubQuery; import org.apache.calcite.rex.RexUtil; -import org.apache.calcite.rex.RexVisitorImpl; import org.apache.calcite.rex.RexWindowBound; import org.apache.calcite.schema.ModifiableTable; import org.apache.calcite.schema.ModifiableView; @@ -85,6 +93,7 @@ import org.apache.calcite.sql.SemiJoinType; import org.apache.calcite.sql.SqlAggFunction; import org.apache.calcite.sql.SqlBasicCall; import org.apache.calcite.sql.SqlCall; +import org.apache.calcite.sql.SqlCallBinding; import org.apache.calcite.sql.SqlDataTypeSpec; import org.apache.calcite.sql.SqlDelete; import org.apache.calcite.sql.SqlDynamicParam; @@ -102,10 +111,12 @@ import org.apache.calcite.sql.SqlNodeList; import org.apache.calcite.sql.SqlNumericLiteral; import org.apache.calcite.sql.SqlOperator; import org.apache.calcite.sql.SqlOperatorTable; +import org.apache.calcite.sql.SqlOrderBy; import org.apache.calcite.sql.SqlSampleSpec; import org.apache.calcite.sql.SqlSelect; import org.apache.calcite.sql.SqlSelectKeyword; import org.apache.calcite.sql.SqlSetOperator; +import org.apache.calcite.sql.SqlUnnestOperator; import org.apache.calcite.sql.SqlUpdate; import org.apache.calcite.sql.SqlUtil; import org.apache.calcite.sql.SqlValuesOperator; @@ -138,8 +149,10 @@ import org.apache.calcite.sql.validate.SqlValidatorImpl; import org.apache.calcite.sql.validate.SqlValidatorNamespace; import org.apache.calcite.sql.validate.SqlValidatorScope; import org.apache.calcite.sql.validate.SqlValidatorUtil; +import org.apache.calcite.tools.RelBuilder; import org.apache.calcite.util.ImmutableBitSet; import org.apache.calcite.util.ImmutableIntList; +import org.apache.calcite.util.Litmus; import org.apache.calcite.util.NlsString; import org.apache.calcite.util.NumberUtil; import org.apache.calcite.util.Pair; @@ -158,15 +171,16 @@ import com.google.common.collect.Sets; import java.lang.reflect.Type; import java.math.BigDecimal; import java.util.AbstractList; +import java.util.ArrayDeque; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.Deque; import java.util.EnumSet; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; -import java.util.Stack; import java.util.TreeSet; import java.util.logging.Level; import java.util.logging.Logger; @@ -178,7 +192,7 @@ import static org.apache.calcite.util.Static.RESOURCE; * OVERRIDE POINT: * - getInSubqueryThreshold(), was `20`, now `Integer.MAX_VALUE` * - isTrimUnusedFields(), override to false - * - AggConverter.visit(SqlCall), skip column reading for COUNT(COL), for https://jirap.corp.ebay.com/browse/KYLIN-104 + * - AggConverter.translateAgg(...), skip column reading for COUNT(COL), for https://jirap.corp.ebay.com/browse/KYLIN-104 */ /** @@ -190,3851 +204,4825 @@ import static org.apache.calcite.util.Static.RESOURCE; * {@link #convertExpression(SqlNode)}. */ public class SqlToRelConverter { - //~ Static fields/initializers --------------------------------------------- - - protected static final Logger SQL2REL_LOGGER = CalciteTrace.getSqlToRelTracer(); - - private static final BigDecimal TWO = BigDecimal.valueOf(2L); - - //~ Instance fields -------------------------------------------------------- - - protected final SqlValidator validator; - protected final RexBuilder rexBuilder; - protected final Prepare.CatalogReader catalogReader; - protected final RelOptCluster cluster; - private DefaultValueFactory defaultValueFactory; - private SubqueryConverter subqueryConverter; - protected final List leaves = new ArrayList<>(); - private final List dynamicParamSqlNodes = new ArrayList<>(); - private final SqlOperatorTable opTab; - private boolean shouldConvertTableAccess; - protected final RelDataTypeFactory typeFactory; - private final SqlNodeToRexConverter exprConverter; - private boolean decorrelationEnabled; - private boolean trimUnusedFields; - private boolean shouldCreateValuesRel; - private boolean isExplain; - private int nDynamicParamsInExplain; - - /** - * Fields used in name resolution for correlated subqueries. - */ - private final Map mapCorrelToDeferred = new HashMap<>(); - private int nextCorrel = 0; - - private static final String CORREL_PREFIX = "$cor"; - - /** - * Stack of names of datasets requested by the - * TABLE(SAMPLE(<datasetName>, <query>)) construct. - */ - private final Stack datasetStack = new Stack<>(); - - /** - * Mapping of non-correlated subqueries that have been converted to their - * equivalent constants. Used to avoid re-evaluating the subquery if it's - * already been evaluated. - */ - private final Map mapConvertedNonCorrSubqs = new HashMap<>(); - - public final RelOptTable.ViewExpander viewExpander; - - //~ Constructors ----------------------------------------------------------- - /** - * Creates a converter. - * - * @param viewExpander Preparing statement - * @param validator Validator - * @param catalogReader Schema - * @param planner Planner - * @param rexBuilder Rex builder - * @param convertletTable Expression converter - */ - @Deprecated - // will be removed before 2.0 - public SqlToRelConverter(RelOptTable.ViewExpander viewExpander, SqlValidator validator, Prepare.CatalogReader catalogReader, RelOptPlanner planner, RexBuilder rexBuilder, SqlRexConvertletTable convertletTable) { - this(viewExpander, validator, catalogReader, RelOptCluster.create(planner, rexBuilder), convertletTable); - } - - /* Creates a converter. */ - public SqlToRelConverter(RelOptTable.ViewExpander viewExpander, SqlValidator validator, Prepare.CatalogReader catalogReader, RelOptCluster cluster, SqlRexConvertletTable convertletTable) { - this.viewExpander = viewExpander; - this.opTab = (validator == null) ? SqlStdOperatorTable.instance() : validator.getOperatorTable(); - this.validator = validator; - this.catalogReader = catalogReader; - this.defaultValueFactory = new NullDefaultValueFactory(); - this.subqueryConverter = new NoOpSubqueryConverter(); - this.rexBuilder = cluster.getRexBuilder(); - this.typeFactory = rexBuilder.getTypeFactory(); - this.cluster = Preconditions.checkNotNull(cluster); - this.shouldConvertTableAccess = true; - this.exprConverter = new SqlNodeToRexConverterImpl(convertletTable); - decorrelationEnabled = true; - trimUnusedFields = false; - shouldCreateValuesRel = true; - isExplain = false; - nDynamicParamsInExplain = 0; - } - - //~ Methods ---------------------------------------------------------------- + //~ Static fields/initializers --------------------------------------------- + + protected static final Logger SQL2REL_LOGGER = + CalciteTrace.getSqlToRelTracer(); + + private static final BigDecimal TWO = BigDecimal.valueOf(2L); + + /** Size of the smallest IN list that will be converted to a semijoin to a + * static table. */ + public static final int IN_SUBQUERY_THRESHOLD = 20; + + //~ Instance fields -------------------------------------------------------- + + protected final SqlValidator validator; + protected final RexBuilder rexBuilder; + protected final Prepare.CatalogReader catalogReader; + protected final RelOptCluster cluster; + private DefaultValueFactory defaultValueFactory; + private SubqueryConverter subqueryConverter; + protected final List leaves = new ArrayList<>(); + private final List dynamicParamSqlNodes = new ArrayList<>(); + private final SqlOperatorTable opTab; + private boolean shouldConvertTableAccess; + protected final RelDataTypeFactory typeFactory; + private final SqlNodeToRexConverter exprConverter; + private boolean decorrelationEnabled; + private boolean trimUnusedFields; + private boolean shouldCreateValuesRel; + private boolean isExplain; + private int nDynamicParamsInExplain; + + /** + * Fields used in name resolution for correlated subqueries. + */ + private final Map mapCorrelToDeferred = + new HashMap<>(); + + /** + * Stack of names of datasets requested by the + * TABLE(SAMPLE(<datasetName>, <query>)) construct. + */ + private final Deque datasetStack = new ArrayDeque<>(); + + /** + * Mapping of non-correlated subqueries that have been converted to their + * equivalent constants. Used to avoid re-evaluating the subquery if it's + * already been evaluated. + */ + private final Map mapConvertedNonCorrSubqs = + new HashMap<>(); + + public final RelOptTable.ViewExpander viewExpander; + + /** Whether to expand sub-queries. If false, each sub-query becomes a + * {@link org.apache.calcite.rex.RexSubQuery}. */ + private boolean expand = true; + + //~ Constructors ----------------------------------------------------------- + /** + * Creates a converter. + * + * @param viewExpander Preparing statement + * @param validator Validator + * @param catalogReader Schema + * @param planner Planner + * @param rexBuilder Rex builder + * @param convertletTable Expression converter + */ + @Deprecated // will be removed before 2.0 + public SqlToRelConverter( + RelOptTable.ViewExpander viewExpander, + SqlValidator validator, + Prepare.CatalogReader catalogReader, + RelOptPlanner planner, + RexBuilder rexBuilder, + SqlRexConvertletTable convertletTable) { + this(viewExpander, validator, catalogReader, + RelOptCluster.create(planner, rexBuilder), convertletTable); + } + + /* Creates a converter. */ + public SqlToRelConverter( + RelOptTable.ViewExpander viewExpander, + SqlValidator validator, + Prepare.CatalogReader catalogReader, + RelOptCluster cluster, + SqlRexConvertletTable convertletTable) { + this.viewExpander = viewExpander; + this.opTab = + (validator + == null) ? SqlStdOperatorTable.instance() + : validator.getOperatorTable(); + this.validator = validator; + this.catalogReader = catalogReader; + this.defaultValueFactory = new NullDefaultValueFactory(); + this.subqueryConverter = new NoOpSubqueryConverter(); + this.rexBuilder = cluster.getRexBuilder(); + this.typeFactory = rexBuilder.getTypeFactory(); + this.cluster = Preconditions.checkNotNull(cluster); + this.shouldConvertTableAccess = true; + this.exprConverter = + new SqlNodeToRexConverterImpl(convertletTable); + decorrelationEnabled = true; + trimUnusedFields = false; + shouldCreateValuesRel = true; + isExplain = false; + nDynamicParamsInExplain = 0; + } + + //~ Methods ---------------------------------------------------------------- + + /** + * @return the RelOptCluster in use. + */ + public RelOptCluster getCluster() { + return cluster; + } + + /** + * Returns the row-expression builder. + */ + public RexBuilder getRexBuilder() { + return rexBuilder; + } + + /** + * Returns the number of dynamic parameters encountered during translation; + * this must only be called after {@link #convertQuery}. + * + * @return number of dynamic parameters + */ + public int getDynamicParamCount() { + return dynamicParamSqlNodes.size(); + } + + /** + * Returns the type inferred for a dynamic parameter. + * + * @param index 0-based index of dynamic parameter + * @return inferred type, never null + */ + public RelDataType getDynamicParamType(int index) { + SqlNode sqlNode = dynamicParamSqlNodes.get(index); + if (sqlNode == null) { + throw Util.needToImplement("dynamic param type inference"); + } + return validator.getValidatedNodeType(sqlNode); + } + + /** + * Returns the current count of the number of dynamic parameters in an + * EXPLAIN PLAN statement. + * + * @param increment if true, increment the count + * @return the current count before the optional increment + */ + public int getDynamicParamCountInExplain(boolean increment) { + int retVal = nDynamicParamsInExplain; + if (increment) { + ++nDynamicParamsInExplain; + } + return retVal; + } + + /** + * @return mapping of non-correlated subqueries that have been converted to + * the constants that they evaluate to + */ + public Map getMapConvertedNonCorrSubqs() { + return mapConvertedNonCorrSubqs; + } + + /** + * Adds to the current map of non-correlated converted subqueries the + * elements from another map that contains non-correlated subqueries that + * have been converted by another SqlToRelConverter. + * + * @param alreadyConvertedNonCorrSubqs the other map + */ + public void addConvertedNonCorrSubqs( + Map alreadyConvertedNonCorrSubqs) { + mapConvertedNonCorrSubqs.putAll(alreadyConvertedNonCorrSubqs); + } + + /** + * Set a new DefaultValueFactory. To have any effect, this must be called + * before any convert method. + * + * @param factory new DefaultValueFactory + */ + public void setDefaultValueFactory(DefaultValueFactory factory) { + defaultValueFactory = factory; + } + + /** + * Sets a new SubqueryConverter. To have any effect, this must be called + * before any convert method. + * + * @param converter new SubqueryConverter + */ + public void setSubqueryConverter(SubqueryConverter converter) { + subqueryConverter = converter; + } + + /** + * Indicates that the current statement is part of an EXPLAIN PLAN statement + * + * @param nDynamicParams number of dynamic parameters in the statement + */ + public void setIsExplain(int nDynamicParams) { + isExplain = true; + nDynamicParamsInExplain = nDynamicParams; + } + + /** + * Controls whether table access references are converted to physical rels + * immediately. The optimizer doesn't like leaf rels to have + * {@link Convention#NONE}. However, if we are doing further conversion + * passes (e.g. {@link RelStructuredTypeFlattener}), then we may need to + * defer conversion. To have any effect, this must be called before any + * convert method. + * + * @param enabled true for immediate conversion (the default); false to + * generate logical LogicalTableScan instances + */ + public void enableTableAccessConversion(boolean enabled) { + shouldConvertTableAccess = enabled; + } + + /** + * Controls whether instances of + * {@link org.apache.calcite.rel.logical.LogicalValues} are generated. These + * may not be supported by all physical implementations. To have any effect, + * this must be called before any convert method. + * + * @param enabled true to allow LogicalValues to be generated (the default); + * false to force substitution of Project+OneRow instead + */ + public void enableValuesRelCreation(boolean enabled) { + shouldCreateValuesRel = enabled; + } + + private void checkConvertedType(SqlNode query, RelNode result) { + if (query.isA(SqlKind.DML)) { + return; + } + // Verify that conversion from SQL to relational algebra did + // not perturb any type information. (We can't do this if the + // SQL statement is something like an INSERT which has no + // validator type information associated with its result, + // hence the namespace check above.) + final List validatedFields = + validator.getValidatedNodeType(query).getFieldList(); + final RelDataType validatedRowType = + validator.getTypeFactory().createStructType( + Pair.right(validatedFields), + SqlValidatorUtil.uniquify(Pair.left(validatedFields))); + + final List convertedFields = + result.getRowType().getFieldList().subList(0, validatedFields.size()); + final RelDataType convertedRowType = + validator.getTypeFactory().createStructType(convertedFields); + + if (!RelOptUtil.equal("validated row type", validatedRowType, + "converted row type", convertedRowType, Litmus.IGNORE)) { + throw new AssertionError("Conversion to relational algebra failed to " + + "preserve datatypes:\n" + + "validated type:\n" + + validatedRowType.getFullTypeString() + + "\nconverted type:\n" + + convertedRowType.getFullTypeString() + + "\nrel:\n" + + RelOptUtil.toString(result)); + } + } + + public RelNode flattenTypes( + RelNode rootRel, + boolean restructure) { + RelStructuredTypeFlattener typeFlattener = + new RelStructuredTypeFlattener(rexBuilder, createToRelContext()); + return typeFlattener.rewrite(rootRel, restructure); + } + + /** + * If subquery is correlated and decorrelation is enabled, performs + * decorrelation. + * + * @param query Query + * @param rootRel Root relational expression + * @return New root relational expression after decorrelation + */ + public RelNode decorrelate(SqlNode query, RelNode rootRel) { + if (!enableDecorrelation()) { + return rootRel; + } + final RelNode result = decorrelateQuery(rootRel); + if (result != rootRel) { + checkConvertedType(query, result); + } + return result; + } + + /** + * Walks over a tree of relational expressions, replacing each + * {@link RelNode} with a 'slimmed down' relational expression that projects + * only the fields required by its consumer. + * + *

This may make things easier for the optimizer, by removing crud that + * would expand the search space, but is difficult for the optimizer itself + * to do it, because optimizer rules must preserve the number and type of + * fields. Hence, this transform that operates on the entire tree, similar + * to the {@link RelStructuredTypeFlattener type-flattening transform}. + * + *

Currently this functionality is disabled in farrago/luciddb; the + * default implementation of this method does nothing. + * + * @param ordered Whether the relational expression must produce results in + * a particular order (typically because it has an ORDER BY at top level) + * @param rootRel Relational expression that is at the root of the tree + * @return Trimmed relational expression + */ + public RelNode trimUnusedFields(boolean ordered, RelNode rootRel) { + // Trim fields that are not used by their consumer. + if (isTrimUnusedFields()) { + final RelFieldTrimmer trimmer = newFieldTrimmer(); + final List collations = + rootRel.getTraitSet().getTraits(RelCollationTraitDef.INSTANCE); + rootRel = trimmer.trim(rootRel); + if (!ordered + && collations != null + && !collations.isEmpty() + && !collations.equals(ImmutableList.of(RelCollations.EMPTY))) { + final RelTraitSet traitSet = rootRel.getTraitSet() + .replace(RelCollationTraitDef.INSTANCE, collations); + rootRel = rootRel.copy(traitSet, rootRel.getInputs()); + } + boolean dumpPlan = SQL2REL_LOGGER.isLoggable(Level.FINE); + if (dumpPlan) { + SQL2REL_LOGGER.fine( + RelOptUtil.dumpPlan( + "Plan after trimming unused fields", + rootRel, + false, + SqlExplainLevel.EXPPLAN_ATTRIBUTES)); + } + } + return rootRel; + } + + /** + * Creates a RelFieldTrimmer. + * + * @return Field trimmer + */ + protected RelFieldTrimmer newFieldTrimmer() { + final RelBuilder relBuilder = + RelFactories.LOGICAL_BUILDER.create(cluster, null); + return new RelFieldTrimmer(validator, relBuilder); + } + + /** + * Converts an unvalidated query's parse tree into a relational expression. + * + * @param query Query to convert + * @param needsValidation Whether to validate the query before converting; + * false if the query has already been + * validated. + * @param top Whether the query is top-level, say if its result + * will become a JDBC result set; false if + * the query will be part of a view. + */ + public RelRoot convertQuery( + SqlNode query, + final boolean needsValidation, + final boolean top) { + if (needsValidation) { + query = validator.validate(query); + } - /** - * @return the RelOptCluster in use. - */ - public RelOptCluster getCluster() { - return cluster; + RelNode result = convertQueryRecursive(query, top, null).rel; + if (top) { + if (isStream(query)) { + result = new LogicalDelta(cluster, result.getTraitSet(), result); + } + } + RelCollation collation = RelCollations.EMPTY; + if (!query.isA(SqlKind.DML)) { + if (isOrdered(query)) { + collation = requiredCollation(result); + } + } + checkConvertedType(query, result); + + boolean dumpPlan = SQL2REL_LOGGER.isLoggable(Level.FINE); + if (dumpPlan) { + SQL2REL_LOGGER.fine( + RelOptUtil.dumpPlan( + "Plan after converting SqlNode to RelNode", + result, + false, + SqlExplainLevel.EXPPLAN_ATTRIBUTES)); } - /** - * Returns the row-expression builder. - */ - public RexBuilder getRexBuilder() { - return rexBuilder; + final RelDataType validatedRowType = validator.getValidatedNodeType(query); + return RelRoot.of(result, validatedRowType, query.getKind()) + .withCollation(collation); + } + + private static boolean isStream(SqlNode query) { + return query instanceof SqlSelect + && ((SqlSelect) query).isKeywordPresent(SqlSelectKeyword.STREAM); + } + + public static boolean isOrdered(SqlNode query) { + switch (query.getKind()) { + case SELECT: + return ((SqlSelect) query).getOrderList() != null + && ((SqlSelect) query).getOrderList().size() > 0; + case WITH: + return isOrdered(((SqlWith) query).body); + case ORDER_BY: + return ((SqlOrderBy) query).orderList.size() > 0; + default: + return false; } + } - /** - * Returns the number of dynamic parameters encountered during translation; - * this must only be called after {@link #convertQuery}. - * - * @return number of dynamic parameters - */ - public int getDynamicParamCount() { - return dynamicParamSqlNodes.size(); + private RelCollation requiredCollation(RelNode r) { + if (r instanceof Sort) { + return ((Sort) r).collation; + } + if (r instanceof Project) { + return requiredCollation(((Project) r).getInput()); + } + if (r instanceof Delta) { + return requiredCollation(((Delta) r).getInput()); + } + throw new AssertionError(); + } + + /** + * Converts a SELECT statement's parse tree into a relational expression. + */ + public RelNode convertSelect(SqlSelect select, boolean top) { + final SqlValidatorScope selectScope = validator.getWhereScope(select); + final Blackboard bb = createBlackboard(selectScope, null, top); + convertSelectImpl(bb, select); + return bb.root; + } + + /** + * Factory method for creating translation workspace. + */ + protected Blackboard createBlackboard(SqlValidatorScope scope, + Map nameToNodeMap, boolean top) { + return new Blackboard(scope, nameToNodeMap, top); + } + + /** + * Implementation of {@link #convertSelect(SqlSelect, boolean)}; + * derived class may override. + */ + protected void convertSelectImpl( + final Blackboard bb, + SqlSelect select) { + convertFrom( + bb, + select.getFrom()); + convertWhere( + bb, + select.getWhere()); + + final List orderExprList = new ArrayList<>(); + final List collationList = new ArrayList<>(); + gatherOrderExprs( + bb, + select, + select.getOrderList(), + orderExprList, + collationList); + final RelCollation collation = + cluster.traitSet().canonize(RelCollations.of(collationList)); + + if (validator.isAggregate(select)) { + convertAgg( + bb, + select, + orderExprList); + } else { + convertSelectList( + bb, + select, + orderExprList); } - /** - * Returns the type inferred for a dynamic parameter. - * - * @param index 0-based index of dynamic parameter - * @return inferred type, never null - */ - public RelDataType getDynamicParamType(int index) { - SqlNode sqlNode = dynamicParamSqlNodes.get(index); - if (sqlNode == null) { - throw Util.needToImplement("dynamic param type inference"); - } - return validator.getValidatedNodeType(sqlNode); + if (select.isDistinct()) { + distinctify(bb, true); + } + convertOrder( + select, bb, collation, orderExprList, select.getOffset(), + select.getFetch()); + bb.setRoot(bb.root, true); + } + + /** + * Having translated 'SELECT ... FROM ... [GROUP BY ...] [HAVING ...]', adds + * a relational expression to make the results unique. + * + *

If the SELECT clause contains duplicate expressions, adds + * {@link org.apache.calcite.rel.logical.LogicalProject}s so that we are + * grouping on the minimal set of keys. The performance gain isn't huge, but + * it is difficult to detect these duplicate expressions later. + * + * @param bb Blackboard + * @param checkForDupExprs Check for duplicate expressions + */ + private void distinctify( + Blackboard bb, + boolean checkForDupExprs) { + // Look for duplicate expressions in the project. + // Say we have 'select x, y, x, z'. + // Then dups will be {[2, 0]} + // and oldToNew will be {[0, 0], [1, 1], [2, 0], [3, 2]} + RelNode rel = bb.root; + if (checkForDupExprs && (rel instanceof LogicalProject)) { + LogicalProject project = (LogicalProject) rel; + final List projectExprs = project.getProjects(); + final List origins = new ArrayList<>(); + int dupCount = 0; + for (int i = 0; i < projectExprs.size(); i++) { + int x = findExpr(projectExprs.get(i), projectExprs, i); + if (x >= 0) { + origins.add(x); + ++dupCount; + } else { + origins.add(i); + } + } + if (dupCount == 0) { + distinctify(bb, false); + return; + } + + final Map squished = Maps.newHashMap(); + final List fields = rel.getRowType().getFieldList(); + final List> newProjects = Lists.newArrayList(); + for (int i = 0; i < fields.size(); i++) { + if (origins.get(i) == i) { + squished.put(i, newProjects.size()); + newProjects.add(RexInputRef.of2(i, fields)); + } + } + rel = + LogicalProject.create(rel, Pair.left(newProjects), + Pair.right(newProjects)); + bb.root = rel; + distinctify(bb, false); + rel = bb.root; + + // Create the expressions to reverse the mapping. + // Project($0, $1, $0, $2). + final List> undoProjects = Lists.newArrayList(); + for (int i = 0; i < fields.size(); i++) { + final int origin = origins.get(i); + RelDataTypeField field = fields.get(i); + undoProjects.add( + Pair.of( + (RexNode) new RexInputRef( + squished.get(origin), field.getType()), + field.getName())); + } + + rel = + LogicalProject.create(rel, Pair.left(undoProjects), + Pair.right(undoProjects)); + bb.setRoot( + rel, + false); + + return; } - /** - * Returns the current count of the number of dynamic parameters in an - * EXPLAIN PLAN statement. - * - * @param increment if true, increment the count - * @return the current count before the optional increment - */ - public int getDynamicParamCountInExplain(boolean increment) { - int retVal = nDynamicParamsInExplain; - if (increment) { - ++nDynamicParamsInExplain; - } - return retVal; + // Usual case: all of the expressions in the SELECT clause are + // different. + final ImmutableBitSet groupSet = + ImmutableBitSet.range(rel.getRowType().getFieldCount()); + rel = + createAggregate(bb, false, groupSet, ImmutableList.of(groupSet), + ImmutableList.of()); + + bb.setRoot( + rel, + false); + } + + private int findExpr(RexNode seek, List exprs, int count) { + for (int i = 0; i < count; i++) { + RexNode expr = exprs.get(i); + if (expr.toString().equals(seek.toString())) { + return i; + } + } + return -1; + } + + /** + * Converts a query's ORDER BY clause, if any. + * + * @param select Query + * @param bb Blackboard + * @param collation Collation list + * @param orderExprList Method populates this list with orderBy expressions + * not present in selectList + * @param offset Expression for number of rows to discard before + * returning first row + * @param fetch Expression for number of rows to fetch + */ + protected void convertOrder( + SqlSelect select, + Blackboard bb, + RelCollation collation, + List orderExprList, + SqlNode offset, + SqlNode fetch) { + if (select.getOrderList() == null + || select.getOrderList().getList().isEmpty()) { + assert collation.getFieldCollations().isEmpty(); + if ((offset == null + || ((SqlLiteral) offset).bigDecimalValue().equals(BigDecimal.ZERO)) + && fetch == null) { + return; + } } - /** - * @return mapping of non-correlated subqueries that have been converted to - * the constants that they evaluate to - */ - public Map getMapConvertedNonCorrSubqs() { - return mapConvertedNonCorrSubqs; + // Create a sorter using the previously constructed collations. + bb.setRoot( + LogicalSort.create(bb.root, collation, + offset == null ? null : convertExpression(offset), + fetch == null ? null : convertExpression(fetch)), + false); + + // If extra expressions were added to the project list for sorting, + // add another project to remove them. But make the collation empty, because + // we can't represent the real collation. + // + // If it is the top node, use the real collation, but don't trim fields. + if (orderExprList.size() > 0 && !bb.top) { + final List exprs = new ArrayList<>(); + final RelDataType rowType = bb.root.getRowType(); + final int fieldCount = + rowType.getFieldCount() - orderExprList.size(); + for (int i = 0; i < fieldCount; i++) { + exprs.add(rexBuilder.makeInputRef(bb.root, i)); + } + bb.setRoot( + LogicalProject.create(bb.root, exprs, + rowType.getFieldNames().subList(0, fieldCount)), + false); + } + } + + /** + * Returns whether a given node contains a {@link SqlInOperator}. + * + * @param node a RexNode tree + */ + private static boolean containsInOperator( + SqlNode node) { + try { + SqlVisitor visitor = + new SqlBasicVisitor() { + public Void visit(SqlCall call) { + if (call.getOperator() instanceof SqlInOperator) { + throw new Util.FoundOne(call); + } + return super.visit(call); + } + }; + node.accept(visitor); + return false; + } catch (Util.FoundOne e) { + Util.swallow(e, null); + return true; } + } + + /** + * Push down all the NOT logical operators into any IN/NOT IN operators. + * + * @param sqlNode the root node from which to look for NOT operators + * @return the transformed SqlNode representation with NOT pushed down. + */ + private static SqlNode pushDownNotForIn(SqlNode sqlNode) { + if ((sqlNode instanceof SqlCall) && containsInOperator(sqlNode)) { + SqlCall sqlCall = (SqlCall) sqlNode; + if ((sqlCall.getOperator() == SqlStdOperatorTable.AND) + || (sqlCall.getOperator() == SqlStdOperatorTable.OR)) { + SqlNode[] sqlOperands = ((SqlBasicCall) sqlCall).operands; + for (int i = 0; i < sqlOperands.length; i++) { + sqlOperands[i] = pushDownNotForIn(sqlOperands[i]); + } + return sqlNode; + } else if (sqlCall.getOperator() == SqlStdOperatorTable.NOT) { + SqlNode childNode = sqlCall.operand(0); + assert childNode instanceof SqlCall; + SqlBasicCall childSqlCall = (SqlBasicCall) childNode; + if (childSqlCall.getOperator() == SqlStdOperatorTable.AND) { + SqlNode[] andOperands = childSqlCall.getOperands(); + SqlNode[] orOperands = new SqlNode[andOperands.length]; + for (int i = 0; i < orOperands.length; i++) { + orOperands[i] = + SqlStdOperatorTable.NOT.createCall( + SqlParserPos.ZERO, + andOperands[i]); + } + for (int i = 0; i < orOperands.length; i++) { + orOperands[i] = pushDownNotForIn(orOperands[i]); + } + return SqlStdOperatorTable.OR.createCall(SqlParserPos.ZERO, + orOperands[0], orOperands[1]); + } else if (childSqlCall.getOperator() == SqlStdOperatorTable.OR) { + SqlNode[] orOperands = childSqlCall.getOperands(); + SqlNode[] andOperands = new SqlNode[orOperands.length]; + for (int i = 0; i < andOperands.length; i++) { + andOperands[i] = + SqlStdOperatorTable.NOT.createCall( + SqlParserPos.ZERO, + orOperands[i]); + } + for (int i = 0; i < andOperands.length; i++) { + andOperands[i] = pushDownNotForIn(andOperands[i]); + } + return SqlStdOperatorTable.AND.createCall(SqlParserPos.ZERO, + andOperands[0], andOperands[1]); + } else if (childSqlCall.getOperator() == SqlStdOperatorTable.NOT) { + SqlNode[] notOperands = childSqlCall.getOperands(); + assert notOperands.length == 1; + return pushDownNotForIn(notOperands[0]); + } else if (childSqlCall.getOperator() instanceof SqlInOperator) { + SqlNode[] inOperands = childSqlCall.getOperands(); + SqlInOperator inOp = + (SqlInOperator) childSqlCall.getOperator(); + if (inOp.isNotIn()) { + return SqlStdOperatorTable.IN.createCall( + SqlParserPos.ZERO, + inOperands[0], + inOperands[1]); + } else { + return SqlStdOperatorTable.NOT_IN.createCall( + SqlParserPos.ZERO, + inOperands[0], + inOperands[1]); + } + } else { + // childSqlCall is "leaf" node in a logical expression tree + // (only considering AND, OR, NOT) + return sqlNode; + } + } else { + // sqlNode is "leaf" node in a logical expression tree + // (only considering AND, OR, NOT) + return sqlNode; + } + } else { + // tree rooted at sqlNode does not contain inOperator + return sqlNode; + } + } + + /** + * Converts a WHERE clause. + * + * @param bb Blackboard + * @param where WHERE clause, may be null + */ + private void convertWhere( + final Blackboard bb, + final SqlNode where) { + if (where == null) { + return; + } + SqlNode newWhere = pushDownNotForIn(where); + replaceSubqueries(bb, newWhere, RelOptUtil.Logic.UNKNOWN_AS_FALSE); + final RexNode convertedWhere = bb.convertExpression(newWhere); - /** - * Adds to the current map of non-correlated converted subqueries the - * elements from another map that contains non-correlated subqueries that - * have been converted by another SqlToRelConverter. - * - * @param alreadyConvertedNonCorrSubqs the other map - */ - public void addConvertedNonCorrSubqs(Map alreadyConvertedNonCorrSubqs) { - mapConvertedNonCorrSubqs.putAll(alreadyConvertedNonCorrSubqs); + // only allocate filter if the condition is not TRUE + if (convertedWhere.isAlwaysTrue()) { + return; } - /** - * Set a new DefaultValueFactory. To have any effect, this must be called - * before any convert method. - * - * @param factory new DefaultValueFactory - */ - public void setDefaultValueFactory(DefaultValueFactory factory) { - defaultValueFactory = factory; + final RelNode filter = RelOptUtil.createFilter(bb.root, convertedWhere); + final RelNode r; + final CorrelationUse p = getCorrelationUse(bb, filter); + if (p != null) { + assert p.r instanceof Filter; + Filter f = (Filter) p.r; + r = LogicalFilter.create(f.getInput(), f.getCondition(), + ImmutableSet.of(p.id)); + } else { + r = filter; } - /** - * Sets a new SubqueryConverter. To have any effect, this must be called - * before any convert method. - * - * @param converter new SubqueryConverter - */ - public void setSubqueryConverter(SubqueryConverter converter) { - subqueryConverter = converter; + bb.setRoot(r, false); + } + + private void replaceSubqueries( + final Blackboard bb, + final SqlNode expr, + RelOptUtil.Logic logic) { + findSubqueries(bb, expr, logic, false); + for (SubQuery node : bb.subqueryList) { + substituteSubquery(bb, node); } + } - /** - * Indicates that the current statement is part of an EXPLAIN PLAN statement - * - * @param nDynamicParams number of dynamic parameters in the statement - */ - public void setIsExplain(int nDynamicParams) { - isExplain = true; - nDynamicParamsInExplain = nDynamicParams; + private void substituteSubquery(Blackboard bb, SubQuery subQuery) { + final RexNode expr = subQuery.expr; + if (expr != null) { + // Already done. + return; } - /** - * Controls whether table access references are converted to physical rels - * immediately. The optimizer doesn't like leaf rels to have - * {@link Convention#NONE}. However, if we are doing further conversion - * passes (e.g. {@link RelStructuredTypeFlattener}), then we may need to - * defer conversion. To have any effect, this must be called before any - * convert method. - * - * @param enabled true for immediate conversion (the default); false to - * generate logical LogicalTableScan instances - */ - public void enableTableAccessConversion(boolean enabled) { - shouldConvertTableAccess = enabled; + final SqlBasicCall call; + final RelNode rel; + final SqlNode query; + final Pair converted; + switch (subQuery.node.getKind()) { + case CURSOR: + convertCursor(bb, subQuery); + return; + + case MULTISET_QUERY_CONSTRUCTOR: + case MULTISET_VALUE_CONSTRUCTOR: + case ARRAY_QUERY_CONSTRUCTOR: + rel = convertMultisets(ImmutableList.of(subQuery.node), bb); + subQuery.expr = bb.register(rel, JoinRelType.INNER); + return; + + case IN: + call = (SqlBasicCall) subQuery.node; + query = call.operand(1); + if (!expand && !(query instanceof SqlNodeList)) { + return; + } + final SqlNode leftKeyNode = call.operand(0); + + final List leftKeys; + switch (leftKeyNode.getKind()) { + case ROW: + leftKeys = Lists.newArrayList(); + for (SqlNode sqlExpr : ((SqlBasicCall) leftKeyNode).getOperandList()) { + leftKeys.add(bb.convertExpression(sqlExpr)); + } + break; + default: + leftKeys = ImmutableList.of(bb.convertExpression(leftKeyNode)); + } + + final boolean isNotIn = ((SqlInOperator) call.getOperator()).isNotIn(); + if (query instanceof SqlNodeList) { + SqlNodeList valueList = (SqlNodeList) query; + if (!containsNullLiteral(valueList) + && valueList.size() < getInSubqueryThreshold()) { + // We're under the threshold, so convert to OR. + subQuery.expr = + convertInToOr( + bb, + leftKeys, + valueList, + isNotIn); + return; + } + + // Otherwise, let convertExists translate + // values list into an inline table for the + // reference to Q below. + } + + // Project out the search columns from the left side + + // Q1: + // "select from emp where emp.deptno in (select col1 from T)" + // + // is converted to + // + // "select from + // emp inner join (select distinct col1 from T)) q + // on emp.deptno = q.col1 + // + // Q2: + // "select from emp where emp.deptno not in (Q)" + // + // is converted to + // + // "select from + // emp left outer join (select distinct col1, TRUE from T) q + // on emp.deptno = q.col1 + // where emp.deptno <> null + // and q.indicator <> TRUE" + // + final boolean outerJoin = bb.subqueryNeedsOuterJoin + || isNotIn + || subQuery.logic == RelOptUtil.Logic.TRUE_FALSE_UNKNOWN; + final RelDataType targetRowType = + SqlTypeUtil.promoteToRowType(typeFactory, + validator.getValidatedNodeType(leftKeyNode), null); + converted = + convertExists(query, RelOptUtil.SubqueryType.IN, subQuery.logic, + outerJoin, targetRowType); + if (converted.right) { + // Generate + // emp CROSS JOIN (SELECT COUNT(*) AS c, + // COUNT(deptno) AS ck FROM dept) + final RelDataType longType = + typeFactory.createSqlType(SqlTypeName.BIGINT); + final RelNode seek = converted.left.getInput(0); // fragile + final int keyCount = leftKeys.size(); + final List args = ImmutableIntList.range(0, keyCount); + LogicalAggregate aggregate = + LogicalAggregate.create(seek, false, ImmutableBitSet.of(), null, + ImmutableList.of( + AggregateCall.create(SqlStdOperatorTable.COUNT, false, + ImmutableList.of(), -1, longType, null), + AggregateCall.create(SqlStdOperatorTable.COUNT, false, + args, -1, longType, null))); + LogicalJoin join = + LogicalJoin.create(bb.root, aggregate, rexBuilder.makeLiteral(true), + ImmutableSet.of(), JoinRelType.INNER); + bb.setRoot(join, false); + } + RexNode rex = + bb.register(converted.left, + outerJoin ? JoinRelType.LEFT : JoinRelType.INNER, leftKeys); + + subQuery.expr = translateIn(subQuery, bb.root, rex); + if (isNotIn) { + subQuery.expr = + rexBuilder.makeCall(SqlStdOperatorTable.NOT, subQuery.expr); + } + return; + + case EXISTS: + // "select from emp where exists (select a from T)" + // + // is converted to the following if the subquery is correlated: + // + // "select from emp left outer join (select AGG_TRUE() as indicator + // from T group by corr_var) q where q.indicator is true" + // + // If there is no correlation, the expression is replaced with a + // boolean indicating whether the subquery returned 0 or >= 1 row. + call = (SqlBasicCall) subQuery.node; + query = call.operand(0); + if (!expand) { + return; + } + converted = convertExists(query, RelOptUtil.SubqueryType.EXISTS, + subQuery.logic, true, null); + assert !converted.right; + if (convertNonCorrelatedSubQuery(subQuery, bb, converted.left, true)) { + return; + } + subQuery.expr = bb.register(converted.left, JoinRelType.LEFT); + return; + + case SCALAR_QUERY: + // Convert the subquery. If it's non-correlated, convert it + // to a constant expression. + if (!expand) { + return; + } + call = (SqlBasicCall) subQuery.node; + query = call.operand(0); + converted = convertExists(query, RelOptUtil.SubqueryType.SCALAR, + subQuery.logic, true, null); + assert !converted.right; + if (convertNonCorrelatedSubQuery(subQuery, bb, converted.left, false)) { + return; + } + rel = convertToSingleValueSubq(query, converted.left); + subQuery.expr = bb.register(rel, JoinRelType.LEFT); + return; + + case SELECT: + // This is used when converting multiset queries: + // + // select * from unnest(select multiset[deptno] from emps); + // + converted = convertExists(subQuery.node, RelOptUtil.SubqueryType.SCALAR, + subQuery.logic, true, null); + assert !converted.right; + subQuery.expr = bb.register(converted.left, JoinRelType.LEFT); + return; + + default: + throw Util.newInternal("unexpected kind of subquery :" + subQuery.node); + } + } + + private RexNode translateIn(SubQuery subQuery, RelNode root, + final RexNode rex) { + switch (subQuery.logic) { + case TRUE: + return rexBuilder.makeLiteral(true); + + case UNKNOWN_AS_FALSE: + assert rex instanceof RexRangeRef; + final int fieldCount = rex.getType().getFieldCount(); + RexNode rexNode = rexBuilder.makeFieldAccess(rex, fieldCount - 1); + rexNode = rexBuilder.makeCall(SqlStdOperatorTable.IS_TRUE, rexNode); + + // Then append the IS NOT NULL(leftKeysForIn). + // + // RexRangeRef contains the following fields: + // leftKeysForIn, + // rightKeysForIn (the original subquery select list), + // nullIndicator + // + // The first two lists contain the same number of fields. + final int k = (fieldCount - 1) / 2; + for (int i = 0; i < k; i++) { + rexNode = + rexBuilder.makeCall( + SqlStdOperatorTable.AND, + rexNode, + rexBuilder.makeCall( + SqlStdOperatorTable.IS_NOT_NULL, + rexBuilder.makeFieldAccess(rex, i))); + } + return rexNode; + + case TRUE_FALSE_UNKNOWN: + case UNKNOWN_AS_TRUE: + // select e.deptno, + // case + // when ct.c = 0 then false + // when dt.i is not null then true + // when e.deptno is null then null + // when ct.ck < ct.c then null + // else false + // end + // from e + // cross join (select count(*) as c, count(deptno) as ck from v) as ct + // left join (select distinct deptno, true as i from v) as dt + // on e.deptno = dt.deptno + final Join join = (Join) root; + final Project left = (Project) join.getLeft(); + final RelNode leftLeft = ((Join) left.getInput()).getLeft(); + final int leftLeftCount = leftLeft.getRowType().getFieldCount(); + final RelDataType nullableBooleanType = + typeFactory.createTypeWithNullability( + typeFactory.createSqlType(SqlTypeName.BOOLEAN), true); + final RelDataType longType = + typeFactory.createSqlType(SqlTypeName.BIGINT); + final RexNode cRef = rexBuilder.makeInputRef(root, leftLeftCount); + final RexNode ckRef = rexBuilder.makeInputRef(root, leftLeftCount + 1); + final RexNode iRef = + rexBuilder.makeInputRef(root, root.getRowType().getFieldCount() - 1); + + final RexLiteral zero = + rexBuilder.makeExactLiteral(BigDecimal.ZERO, longType); + final RexLiteral trueLiteral = rexBuilder.makeLiteral(true); + final RexLiteral falseLiteral = rexBuilder.makeLiteral(false); + final RexNode unknownLiteral = + rexBuilder.makeNullLiteral(SqlTypeName.BOOLEAN); + + final ImmutableList.Builder args = ImmutableList.builder(); + args.add(rexBuilder.makeCall(SqlStdOperatorTable.EQUALS, cRef, zero), + falseLiteral, + rexBuilder.makeCall(SqlStdOperatorTable.IS_NOT_NULL, iRef), + trueLiteral); + final JoinInfo joinInfo = join.analyzeCondition(); + for (int leftKey : joinInfo.leftKeys) { + final RexNode kRef = rexBuilder.makeInputRef(root, leftKey); + args.add(rexBuilder.makeCall(SqlStdOperatorTable.IS_NULL, kRef), + unknownLiteral); + } + args.add(rexBuilder.makeCall(SqlStdOperatorTable.LESS_THAN, ckRef, cRef), + unknownLiteral, + falseLiteral); + + return rexBuilder.makeCall( + nullableBooleanType, + SqlStdOperatorTable.CASE, + args.build()); + + default: + throw new AssertionError(subQuery.logic); } + } - /** - * Controls whether instances of - * {@link org.apache.calcite.rel.logical.LogicalValues} are generated. These - * may not be supported by all physical implementations. To have any effect, - * this must be called before any convert method. - * - * @param enabled true to allow LogicalValues to be generated (the default); - * false to force substitution of Project+OneRow instead - */ - public void enableValuesRelCreation(boolean enabled) { - shouldCreateValuesRel = enabled; - } - - private void checkConvertedType(SqlNode query, RelNode result) { - if (!query.isA(SqlKind.DML)) { - // Verify that conversion from SQL to relational algebra did - // not perturb any type information. (We can't do this if the - // SQL statement is something like an INSERT which has no - // validator type information associated with its result, - // hence the namespace check above.) - RelDataType convertedRowType = result.getRowType(); - if (!checkConvertedRowType(query, convertedRowType)) { - RelDataType validatedRowType = validator.getValidatedNodeType(query); - validatedRowType = uniquifyFields(validatedRowType); - throw Util.newInternal("Conversion to relational algebra failed to " + "preserve datatypes:\n" + "validated type:\n" + validatedRowType.getFullTypeString() + "\nconverted type:\n" + convertedRowType.getFullTypeString() + "\nrel:\n" + RelOptUtil.toString(result)); - } + private static boolean containsNullLiteral(SqlNodeList valueList) { + for (SqlNode node : valueList.getList()) { + if (node instanceof SqlLiteral) { + SqlLiteral lit = (SqlLiteral) node; + if (lit.getValue() == null) { + return true; } + } } - - public RelNode flattenTypes(RelNode rootRel, boolean restructure) { - RelStructuredTypeFlattener typeFlattener = new RelStructuredTypeFlattener(rexBuilder, createToRelContext()); - return typeFlattener.rewrite(rootRel, restructure); + return false; + } + + /** + * Determines if a subquery is non-correlated and if so, converts it to a + * constant. + * + * @param subQuery the call that references the subquery + * @param bb blackboard used to convert the subquery + * @param converted RelNode tree corresponding to the subquery + * @param isExists true if the subquery is part of an EXISTS expression + * @return if the subquery can be converted to a constant + */ + private boolean convertNonCorrelatedSubQuery( + SubQuery subQuery, + Blackboard bb, + RelNode converted, + boolean isExists) { + SqlCall call = (SqlBasicCall) subQuery.node; + if (subqueryConverter.canConvertSubquery() + && isSubQueryNonCorrelated(converted, bb)) { + // First check if the subquery has already been converted + // because it's a nested subquery. If so, don't re-evaluate + // it again. + RexNode constExpr = mapConvertedNonCorrSubqs.get(call); + if (constExpr == null) { + constExpr = + subqueryConverter.convertSubquery( + call, + this, + isExists, + isExplain); + } + if (constExpr != null) { + subQuery.expr = constExpr; + mapConvertedNonCorrSubqs.put(call, constExpr); + return true; + } + } + return false; + } + + /** + * Converts the RelNode tree for a select statement to a select that + * produces a single value. + * + * @param query the query + * @param plan the original RelNode tree corresponding to the statement + * @return the converted RelNode tree + */ + public RelNode convertToSingleValueSubq( + SqlNode query, + RelNode plan) { + // Check whether query is guaranteed to produce a single value. + if (query instanceof SqlSelect) { + SqlSelect select = (SqlSelect) query; + SqlNodeList selectList = select.getSelectList(); + SqlNodeList groupList = select.getGroup(); + + if ((selectList.size() == 1) + && ((groupList == null) || (groupList.size() == 0))) { + SqlNode selectExpr = selectList.get(0); + if (selectExpr instanceof SqlCall) { + SqlCall selectExprCall = (SqlCall) selectExpr; + if (Util.isSingleValue(selectExprCall)) { + return plan; + } + } + + // If there is a limit with 0 or 1, + // it is ensured to produce a single value + if (select.getFetch() != null + && select.getFetch() instanceof SqlNumericLiteral) { + SqlNumericLiteral limitNum = (SqlNumericLiteral) select.getFetch(); + if (((BigDecimal) limitNum.getValue()).intValue() < 2) { + return plan; + } + } + } + } else if (query instanceof SqlCall) { + // If the query is (values ...), + // it is necessary to look into the operands to determine + // whether SingleValueAgg is necessary + SqlCall exprCall = (SqlCall) query; + if (exprCall.getOperator() + instanceof SqlValuesOperator + && Util.isSingleValue(exprCall)) { + return plan; + } } - /** - * If subquery is correlated and decorrelation is enabled, performs - * decorrelation. - * - * @param query Query - * @param rootRel Root relational expression - * @return New root relational expression after decorrelation - */ - public RelNode decorrelate(SqlNode query, RelNode rootRel) { - if (!enableDecorrelation()) { - return rootRel; - } - final RelNode result = decorrelateQuery(rootRel); - if (result != rootRel) { - checkConvertedType(query, result); - } - return result; + // If not, project SingleValueAgg + return RelOptUtil.createSingleValueAggRel( + cluster, + plan); + } + + /** + * Converts "x IN (1, 2, ...)" to "x=1 OR x=2 OR ...". + * + * @param leftKeys LHS + * @param valuesList RHS + * @param isNotIn is this a NOT IN operator + * @return converted expression + */ + private RexNode convertInToOr( + final Blackboard bb, + final List leftKeys, + SqlNodeList valuesList, + boolean isNotIn) { + final List comparisons = new ArrayList<>(); + for (SqlNode rightVals : valuesList) { + RexNode rexComparison; + if (leftKeys.size() == 1) { + rexComparison = + rexBuilder.makeCall( + SqlStdOperatorTable.EQUALS, + leftKeys.get(0), + rexBuilder.ensureType(leftKeys.get(0).getType(), + bb.convertExpression(rightVals), true)); + } else { + assert rightVals instanceof SqlCall; + final SqlBasicCall call = (SqlBasicCall) rightVals; + assert (call.getOperator() instanceof SqlRowOperator) + && call.operandCount() == leftKeys.size(); + rexComparison = + RexUtil.composeConjunction( + rexBuilder, + Iterables.transform( + Pair.zip(leftKeys, call.getOperandList()), + new Function, RexNode>() { + public RexNode apply(Pair pair) { + return rexBuilder.makeCall(SqlStdOperatorTable.EQUALS, + pair.left, + rexBuilder.ensureType(pair.left.getType(), + bb.convertExpression(pair.right), true)); + } + }), + false); + } + comparisons.add(rexComparison); } - /** - * Walks over a tree of relational expressions, replacing each - * {@link RelNode} with a 'slimmed down' relational expression that projects - * only the fields required by its consumer. - * - *

This may make things easier for the optimizer, by removing crud that - * would expand the search space, but is difficult for the optimizer itself - * to do it, because optimizer rules must preserve the number and type of - * fields. Hence, this transform that operates on the entire tree, similar - * to the {@link RelStructuredTypeFlattener type-flattening transform}. - * - *

Currently this functionality is disabled in farrago/luciddb; the - * default implementation of this method does nothing. - * - * @param rootRel Relational expression that is at the root of the tree - * @return Trimmed relational expression - */ - public RelNode trimUnusedFields(RelNode rootRel) { - // Trim fields that are not used by their consumer. - if (isTrimUnusedFields()) { - final RelFieldTrimmer trimmer = newFieldTrimmer(); - rootRel = trimmer.trim(rootRel); - boolean dumpPlan = SQL2REL_LOGGER.isLoggable(Level.FINE); - if (dumpPlan) { - SQL2REL_LOGGER.fine(RelOptUtil.dumpPlan("Plan after trimming unused fields", rootRel, false, SqlExplainLevel.EXPPLAN_ATTRIBUTES)); - } - } - return rootRel; + RexNode result = + RexUtil.composeDisjunction(rexBuilder, comparisons, true); + assert result != null; + + if (isNotIn) { + result = + rexBuilder.makeCall( + SqlStdOperatorTable.NOT, + result); } - /** - * Creates a RelFieldTrimmer. - * - * @return Field trimmer - */ - protected RelFieldTrimmer newFieldTrimmer() { - return new RelFieldTrimmer(validator); + return result; + } + + /** + * Gets the list size threshold under which {@link #convertInToOr} is used. + * Lists of this size or greater will instead be converted to use a join + * against an inline table + * ({@link org.apache.calcite.rel.logical.LogicalValues}) rather than a + * predicate. A threshold of 0 forces usage of an inline table in all cases; a + * threshold of Integer.MAX_VALUE forces usage of OR in all cases + * + * @return threshold, default {@link #IN_SUBQUERY_THRESHOLD} + */ + protected int getInSubqueryThreshold() { + /* OVERRIDE POINT */ + return Integer.MAX_VALUE; + } + + /** + * Converts an EXISTS or IN predicate into a join. For EXISTS, the subquery + * produces an indicator variable, and the result is a relational expression + * which outer joins that indicator to the original query. After performing + * the outer join, the condition will be TRUE if the EXISTS condition holds, + * NULL otherwise. + * + * @param seek A query, for example 'select * from emp' or + * 'values (1,2,3)' or '('Foo', 34)'. + * @param subqueryType Whether sub-query is IN, EXISTS or scalar + * @param logic Whether the answer needs to be in full 3-valued logic (TRUE, + * FALSE, UNKNOWN) will be required, or whether we can accept an + * approximation (say representing UNKNOWN as FALSE) + * @param needsOuterJoin Whether an outer join is needed + * @return join expression + * @pre extraExpr == null || extraName != null + */ + private Pair convertExists( + SqlNode seek, + RelOptUtil.SubqueryType subqueryType, + RelOptUtil.Logic logic, + boolean needsOuterJoin, + RelDataType targetDataType) { + final SqlValidatorScope seekScope = + (seek instanceof SqlSelect) + ? validator.getSelectScope((SqlSelect) seek) + : null; + final Blackboard seekBb = createBlackboard(seekScope, null, false); + RelNode seekRel = convertQueryOrInList(seekBb, seek, targetDataType); + + return RelOptUtil.createExistsPlan(seekRel, subqueryType, logic, + needsOuterJoin); + } + + private RelNode convertQueryOrInList( + Blackboard bb, + SqlNode seek, + RelDataType targetRowType) { + // NOTE: Once we start accepting single-row queries as row constructors, + // there will be an ambiguity here for a case like X IN ((SELECT Y FROM + // Z)). The SQL standard resolves the ambiguity by saying that a lone + // select should be interpreted as a table expression, not a row + // expression. The semantic difference is that a table expression can + // return multiple rows. + if (seek instanceof SqlNodeList) { + return convertRowValues( + bb, + seek, + ((SqlNodeList) seek).getList(), + false, + targetRowType); + } else { + return convertQueryRecursive(seek, false, null).project(); + } + } + + private RelNode convertRowValues( + Blackboard bb, + SqlNode rowList, + Collection rows, + boolean allowLiteralsOnly, + RelDataType targetRowType) { + // NOTE jvs 30-Apr-2006: We combine all rows consisting entirely of + // literals into a single LogicalValues; this gives the optimizer a smaller + // input tree. For everything else (computed expressions, row + // subqueries), we union each row in as a projection on top of a + // LogicalOneRow. + + final ImmutableList.Builder> tupleList = + ImmutableList.builder(); + final RelDataType rowType; + if (targetRowType != null) { + rowType = targetRowType; + } else { + rowType = + SqlTypeUtil.promoteToRowType( + typeFactory, + validator.getValidatedNodeType(rowList), + null); } - /** - * Converts an unvalidated query's parse tree into a relational expression. - * - * @param query Query to convert - * @param needsValidation Whether to validate the query before converting; - * false if the query has already been - * validated. - * @param top Whether the query is top-level, say if its result - * will become a JDBC result set; false if - * the query will be part of a view. - */ - public RelNode convertQuery(SqlNode query, final boolean needsValidation, final boolean top) { - if (needsValidation) { - query = validator.validate(query); + final List unionInputs = new ArrayList<>(); + for (SqlNode node : rows) { + SqlBasicCall call; + if (isRowConstructor(node)) { + call = (SqlBasicCall) node; + ImmutableList.Builder tuple = ImmutableList.builder(); + for (Ord operand : Ord.zip(call.operands)) { + RexLiteral rexLiteral = + convertLiteralInValuesList( + operand.e, + bb, + rowType, + operand.i); + if ((rexLiteral == null) && allowLiteralsOnly) { + return null; + } + if ((rexLiteral == null) || !shouldCreateValuesRel) { + // fallback to convertRowConstructor + tuple = null; + break; + } + tuple.add(rexLiteral); + } + if (tuple != null) { + tupleList.add(tuple.build()); + continue; + } + } else { + RexLiteral rexLiteral = + convertLiteralInValuesList( + node, + bb, + rowType, + 0); + if ((rexLiteral != null) && shouldCreateValuesRel) { + tupleList.add(ImmutableList.of(rexLiteral)); + continue; + } else { + if ((rexLiteral == null) && allowLiteralsOnly) { + return null; + } } - RelNode result = convertQueryRecursive(query, top, null); - if (top && isStream(query)) { - result = new LogicalDelta(cluster, result.getTraitSet(), result); - } - checkConvertedType(query, result); + // convert "1" to "row(1)" + call = + (SqlBasicCall) SqlStdOperatorTable.ROW.createCall( + SqlParserPos.ZERO, + node); + } + unionInputs.add(convertRowConstructor(bb, call)); + } + LogicalValues values = + LogicalValues.create(cluster, rowType, tupleList.build()); + RelNode resultRel; + if (unionInputs.isEmpty()) { + resultRel = values; + } else { + if (!values.getTuples().isEmpty()) { + unionInputs.add(values); + } + resultRel = LogicalUnion.create(unionInputs, true); + } + leaves.add(resultRel); + return resultRel; + } + + private RexLiteral convertLiteralInValuesList( + SqlNode sqlNode, + Blackboard bb, + RelDataType rowType, + int iField) { + if (!(sqlNode instanceof SqlLiteral)) { + return null; + } + RelDataTypeField field = rowType.getFieldList().get(iField); + RelDataType type = field.getType(); + if (type.isStruct()) { + // null literals for weird stuff like UDT's need + // special handling during type flattening, so + // don't use LogicalValues for those + return null; + } - boolean dumpPlan = SQL2REL_LOGGER.isLoggable(Level.FINE); - if (dumpPlan) { - SQL2REL_LOGGER.fine(RelOptUtil.dumpPlan("Plan after converting SqlNode to RelNode", result, false, SqlExplainLevel.EXPPLAN_ATTRIBUTES)); - } + RexNode literalExpr = + exprConverter.convertLiteral( + bb, + (SqlLiteral) sqlNode); + + if (!(literalExpr instanceof RexLiteral)) { + assert literalExpr.isA(SqlKind.CAST); + RexNode child = ((RexCall) literalExpr).getOperands().get(0); + assert RexLiteral.isNullLiteral(child); - return result; + // NOTE jvs 22-Nov-2006: we preserve type info + // in LogicalValues digest, so it's OK to lose it here + return (RexLiteral) child; } - private static boolean isStream(SqlNode query) { - return query instanceof SqlSelect && ((SqlSelect) query).isKeywordPresent(SqlSelectKeyword.STREAM); + RexLiteral literal = (RexLiteral) literalExpr; + + Comparable value = literal.getValue(); + + if (SqlTypeUtil.isExactNumeric(type) && SqlTypeUtil.hasScale(type)) { + BigDecimal roundedValue = + NumberUtil.rescaleBigDecimal( + (BigDecimal) value, + type.getScale()); + return rexBuilder.makeExactLiteral( + roundedValue, + type); } - protected boolean checkConvertedRowType(SqlNode query, RelDataType convertedRowType) { - RelDataType validatedRowType = validator.getValidatedNodeType(query); - validatedRowType = uniquifyFields(validatedRowType); + if ((value instanceof NlsString) + && (type.getSqlTypeName() == SqlTypeName.CHAR)) { + // pad fixed character type + NlsString unpadded = (NlsString) value; + return rexBuilder.makeCharLiteral( + new NlsString( + Spaces.padRight(unpadded.getValue(), type.getPrecision()), + unpadded.getCharsetName(), + unpadded.getCollation())); + } + return literal; + } - return RelOptUtil.equal("validated row type", validatedRowType, "converted row type", convertedRowType, false); + private boolean isRowConstructor(SqlNode node) { + if (!(node.getKind() == SqlKind.ROW)) { + return false; + } + SqlCall call = (SqlCall) node; + return call.getOperator().getName().equalsIgnoreCase("row"); + } + + /** + * Builds a list of all IN or EXISTS operators + * inside SQL parse tree. Does not traverse inside queries. + * + * @param bb blackboard + * @param node the SQL parse tree + * @param logic Whether the answer needs to be in full 3-valued logic (TRUE, + * FALSE, UNKNOWN) will be required, or whether we can accept + * an approximation (say representing UNKNOWN as FALSE) + * @param registerOnlyScalarSubqueries if set to true and the parse tree + * corresponds to a variation of a select + * node, only register it if it's a scalar + * subquery + */ + private void findSubqueries( + Blackboard bb, + SqlNode node, + RelOptUtil.Logic logic, + boolean registerOnlyScalarSubqueries) { + final SqlKind kind = node.getKind(); + switch (kind) { + case EXISTS: + case SELECT: + case MULTISET_QUERY_CONSTRUCTOR: + case MULTISET_VALUE_CONSTRUCTOR: + case ARRAY_QUERY_CONSTRUCTOR: + case CURSOR: + case SCALAR_QUERY: + if (!registerOnlyScalarSubqueries + || (kind == SqlKind.SCALAR_QUERY)) { + bb.registerSubquery(node, RelOptUtil.Logic.TRUE_FALSE); + } + return; + case IN: + if (((SqlCall) node).getOperator() == SqlStdOperatorTable.NOT_IN) { + logic = logic.negate(); + } + break; + case NOT: + logic = logic.negate(); + break; + } + if (node instanceof SqlCall) { + if (kind == SqlKind.OR + || kind == SqlKind.NOT) { + // It's always correct to outer join subquery with + // containing query; however, when predicates involve Or + // or NOT, outer join might be necessary. + bb.subqueryNeedsOuterJoin = true; + } + for (SqlNode operand : ((SqlCall) node).getOperandList()) { + if (operand != null) { + // In the case of an IN expression, locate scalar + // subqueries so we can convert them to constants + findSubqueries( + bb, + operand, + logic, + kind == SqlKind.IN || registerOnlyScalarSubqueries); + } + } + } else if (node instanceof SqlNodeList) { + for (SqlNode child : (SqlNodeList) node) { + findSubqueries( + bb, + child, + logic, + kind == SqlKind.IN || registerOnlyScalarSubqueries); + } } - protected RelDataType uniquifyFields(RelDataType rowType) { - return validator.getTypeFactory().createStructType(RelOptUtil.getFieldTypeList(rowType), SqlValidatorUtil.uniquify(rowType.getFieldNames())); + // Now that we've located any scalar subqueries inside the IN + // expression, register the IN expression itself. We need to + // register the scalar subqueries first so they can be converted + // before the IN expression is converted. + if (kind == SqlKind.IN) { + if (logic == RelOptUtil.Logic.TRUE_FALSE_UNKNOWN + && !validator.getValidatedNodeType(node).isNullable()) { + logic = RelOptUtil.Logic.UNKNOWN_AS_FALSE; + } + // TODO: This conversion is only valid in the WHERE clause + if (logic == RelOptUtil.Logic.UNKNOWN_AS_FALSE + && !bb.subqueryNeedsOuterJoin) { + logic = RelOptUtil.Logic.TRUE; + } + bb.registerSubquery(node, logic); + } + } + + /** + * Converts an expression from {@link SqlNode} to {@link RexNode} format. + * + * @param node Expression to translate + * @return Converted expression + */ + public RexNode convertExpression( + SqlNode node) { + Map nameToTypeMap = Collections.emptyMap(); + final ParameterScope scope = + new ParameterScope((SqlValidatorImpl) validator, nameToTypeMap); + final Blackboard bb = createBlackboard(scope, null, false); + return bb.convertExpression(node); + } + + /** + * Converts an expression from {@link SqlNode} to {@link RexNode} format, + * mapping identifier references to predefined expressions. + * + * @param node Expression to translate + * @param nameToNodeMap map from String to {@link RexNode}; when an + * {@link SqlIdentifier} is encountered, it is used as a + * key and translated to the corresponding value from + * this map + * @return Converted expression + */ + public RexNode convertExpression( + SqlNode node, + Map nameToNodeMap) { + final Map nameToTypeMap = new HashMap<>(); + for (Map.Entry entry : nameToNodeMap.entrySet()) { + nameToTypeMap.put(entry.getKey(), entry.getValue().getType()); + } + final ParameterScope scope = + new ParameterScope((SqlValidatorImpl) validator, nameToTypeMap); + final Blackboard bb = createBlackboard(scope, nameToNodeMap, false); + return bb.convertExpression(node); + } + + /** + * Converts a non-standard expression. + * + *

This method is an extension-point that derived classes can override. If + * this method returns a null result, the normal expression translation + * process will proceed. The default implementation always returns null. + * + * @param node Expression + * @param bb Blackboard + * @return null to proceed with the usual expression translation process + */ + protected RexNode convertExtendedExpression( + SqlNode node, + Blackboard bb) { + return null; + } + + private RexNode convertOver(Blackboard bb, SqlNode node) { + SqlCall call = (SqlCall) node; + SqlCall aggCall = call.operand(0); + SqlNode windowOrRef = call.operand(1); + final SqlWindow window = + validator.resolveWindow(windowOrRef, bb.scope, true); + // ROW_NUMBER() expects specific kind of framing. + if (aggCall.getOperator() == SqlStdOperatorTable.ROW_NUMBER) { + window.setLowerBound(SqlWindow.createUnboundedPreceding(SqlParserPos.ZERO)); + window.setUpperBound(SqlWindow.createCurrentRow(SqlParserPos.ZERO)); + window.setRows(SqlLiteral.createBoolean(true, SqlParserPos.ZERO)); + } + final SqlNodeList partitionList = window.getPartitionList(); + final ImmutableList.Builder partitionKeys = + ImmutableList.builder(); + for (SqlNode partition : partitionList) { + partitionKeys.add(bb.convertExpression(partition)); + } + RexNode lowerBound = bb.convertExpression(window.getLowerBound()); + RexNode upperBound = bb.convertExpression(window.getUpperBound()); + SqlNodeList orderList = window.getOrderList(); + if ((orderList.size() == 0) && !window.isRows()) { + // A logical range requires an ORDER BY clause. Use the implicit + // ordering of this relation. There must be one, otherwise it would + // have failed validation. + orderList = bb.scope.getOrderList(); + if (orderList == null) { + throw new AssertionError( + "Relation should have sort key for implicit ORDER BY"); + } + } + final ImmutableList.Builder orderKeys = + ImmutableList.builder(); + final Set flags = EnumSet.noneOf(SqlKind.class); + for (SqlNode order : orderList) { + flags.clear(); + RexNode e = bb.convertSortExpression(order, flags); + orderKeys.add(new RexFieldCollation(e, flags)); + } + try { + Util.permAssert(bb.window == null, "already in window agg mode"); + bb.window = window; + RexNode rexAgg = exprConverter.convertCall(bb, aggCall); + rexAgg = + rexBuilder.ensureType( + validator.getValidatedNodeType(call), rexAgg, false); + + // Walk over the tree and apply 'over' to all agg functions. This is + // necessary because the returned expression is not necessarily a call + // to an agg function. For example, AVG(x) becomes SUM(x) / COUNT(x). + final RexShuttle visitor = + new HistogramShuttle( + partitionKeys.build(), orderKeys.build(), + RexWindowBound.create(window.getLowerBound(), lowerBound), + RexWindowBound.create(window.getUpperBound(), upperBound), + window); + return rexAgg.accept(visitor); + } finally { + bb.window = null; + } + } + + /** + * Converts a FROM clause into a relational expression. + * + * @param bb Scope within which to resolve identifiers + * @param from FROM clause of a query. Examples include: + * + *

    + *
  • a single table ("SALES.EMP"), + *
  • an aliased table ("EMP AS E"), + *
  • a list of tables ("EMP, DEPT"), + *
  • an ANSI Join expression ("EMP JOIN DEPT ON EMP.DEPTNO = + * DEPT.DEPTNO"), + *
  • a VALUES clause ("VALUES ('Fred', 20)"), + *
  • a query ("(SELECT * FROM EMP WHERE GENDER = 'F')"), + *
  • or any combination of the above. + *
+ */ + protected void convertFrom( + Blackboard bb, + SqlNode from) { + final SqlCall call; + final SqlNode[] operands; + switch (from.getKind()) { + case AS: + convertFrom(bb, ((SqlCall) from).operand(0)); + return; + + case WITH_ITEM: + convertFrom(bb, ((SqlWithItem) from).query); + return; + + case WITH: + convertFrom(bb, ((SqlWith) from).body); + return; + + case TABLESAMPLE: + operands = ((SqlBasicCall) from).getOperands(); + SqlSampleSpec sampleSpec = SqlLiteral.sampleValue(operands[1]); + if (sampleSpec instanceof SqlSampleSpec.SqlSubstitutionSampleSpec) { + String sampleName = + ((SqlSampleSpec.SqlSubstitutionSampleSpec) sampleSpec) + .getName(); + datasetStack.push(sampleName); + convertFrom(bb, operands[0]); + datasetStack.pop(); + } else if (sampleSpec instanceof SqlSampleSpec.SqlTableSampleSpec) { + SqlSampleSpec.SqlTableSampleSpec tableSampleSpec = + (SqlSampleSpec.SqlTableSampleSpec) sampleSpec; + convertFrom(bb, operands[0]); + RelOptSamplingParameters params = + new RelOptSamplingParameters( + tableSampleSpec.isBernoulli(), + tableSampleSpec.getSamplePercentage(), + tableSampleSpec.isRepeatable(), + tableSampleSpec.getRepeatableSeed()); + bb.setRoot(new Sample(cluster, bb.root, params), false); + } else { + throw Util.newInternal( + "unknown TABLESAMPLE type: " + sampleSpec); + } + return; + + case IDENTIFIER: + final SqlValidatorNamespace fromNamespace = + validator.getNamespace(from).resolve(); + if (fromNamespace.getNode() != null) { + convertFrom(bb, fromNamespace.getNode()); + return; + } + final String datasetName = + datasetStack.isEmpty() ? null : datasetStack.peek(); + boolean[] usedDataset = {false}; + RelOptTable table = + SqlValidatorUtil.getRelOptTable( + fromNamespace, + catalogReader, + datasetName, + usedDataset); + final RelNode tableRel; + if (shouldConvertTableAccess) { + tableRel = toRel(table); + } else { + tableRel = LogicalTableScan.create(cluster, table); + } + bb.setRoot(tableRel, true); + if (usedDataset[0]) { + bb.setDataset(datasetName); + } + return; + + case JOIN: + final SqlJoin join = (SqlJoin) from; + final SqlValidatorScope scope = validator.getJoinScope(from); + final Blackboard fromBlackboard = createBlackboard(scope, null, false); + SqlNode left = join.getLeft(); + SqlNode right = join.getRight(); + final boolean isNatural = join.isNatural(); + final JoinType joinType = join.getJoinType(); + final SqlValidatorScope leftScope = + Util.first(validator.getJoinScope(left), + ((DelegatingScope) bb.scope).getParent()); + final Blackboard leftBlackboard = + createBlackboard(leftScope, null, false); + final SqlValidatorScope rightScope = + Util.first(validator.getJoinScope(right), + ((DelegatingScope) bb.scope).getParent()); + final Blackboard rightBlackboard = + createBlackboard(rightScope, null, false); + convertFrom(leftBlackboard, left); + RelNode leftRel = leftBlackboard.root; + convertFrom(rightBlackboard, right); + RelNode rightRel = rightBlackboard.root; + JoinRelType convertedJoinType = convertJoinType(joinType); + RexNode conditionExp; + final SqlValidatorNamespace leftNamespace = validator.getNamespace(left); + final SqlValidatorNamespace rightNamespace = validator.getNamespace(right); + if (isNatural) { + final RelDataType leftRowType = leftNamespace.getRowType(); + final RelDataType rightRowType = rightNamespace.getRowType(); + final List columnList = + SqlValidatorUtil.deriveNaturalJoinColumnList(leftRowType, + rightRowType); + conditionExp = convertUsing(leftNamespace, rightNamespace, + columnList); + } else { + conditionExp = + convertJoinCondition( + fromBlackboard, + leftNamespace, + rightNamespace, + join.getCondition(), + join.getConditionType(), + leftRel, + rightRel); + } + + final RelNode joinRel = + createJoin( + fromBlackboard, + leftRel, + rightRel, + conditionExp, + convertedJoinType); + bb.setRoot(joinRel, false); + return; + + case SELECT: + case INTERSECT: + case EXCEPT: + case UNION: + final RelNode rel = convertQueryRecursive(from, false, null).project(); + bb.setRoot(rel, true); + return; + + case VALUES: + convertValuesImpl(bb, (SqlCall) from, null); + return; + + case UNNEST: + call = (SqlCall) from; + final SqlNode node = call.operand(0); + final SqlUnnestOperator operator = (SqlUnnestOperator) call.getOperator(); + replaceSubqueries(bb, node, RelOptUtil.Logic.TRUE_FALSE_UNKNOWN); + final RelNode childRel = + RelOptUtil.createProject( + (null != bb.root) ? bb.root : LogicalValues.createOneRow(cluster), + Collections.singletonList(bb.convertExpression(node)), + Collections.singletonList(validator.deriveAlias(node, 0)), + true); + + Uncollect uncollect = + new Uncollect(cluster, cluster.traitSetOf(Convention.NONE), + childRel, operator.withOrdinality); + bb.setRoot(uncollect, true); + return; + + case COLLECTION_TABLE: + call = (SqlCall) from; + + // Dig out real call; TABLE() wrapper is just syntactic. + assert call.getOperandList().size() == 1; + final SqlCall call2 = call.operand(0); + convertCollectionTable(bb, call2); + return; + + default: + throw Util.newInternal("not a join operator " + from); + } + } + + protected void convertCollectionTable( + Blackboard bb, + SqlCall call) { + final SqlOperator operator = call.getOperator(); + if (operator == SqlStdOperatorTable.TABLESAMPLE) { + final String sampleName = + SqlLiteral.stringValue(call.operand(0)); + datasetStack.push(sampleName); + SqlCall cursorCall = call.operand(1); + SqlNode query = cursorCall.operand(0); + RelNode converted = convertQuery(query, false, false).rel; + bb.setRoot(converted, false); + datasetStack.pop(); + return; + } + replaceSubqueries(bb, call, RelOptUtil.Logic.TRUE_FALSE_UNKNOWN); + + // Expand table macro if possible. It's more efficient than + // LogicalTableFunctionScan. + final SqlCallBinding callBinding = + new SqlCallBinding(bb.scope.getValidator(), bb.scope, call); + if (operator instanceof SqlUserDefinedTableMacro) { + final SqlUserDefinedTableMacro udf = + (SqlUserDefinedTableMacro) operator; + final TranslatableTable table = + udf.getTable(typeFactory, callBinding.operands()); + final RelDataType rowType = table.getRowType(typeFactory); + RelOptTable relOptTable = RelOptTableImpl.create(null, rowType, table); + RelNode converted = toRel(relOptTable); + bb.setRoot(converted, true); + return; } - /** - * Converts a SELECT statement's parse tree into a relational expression. - */ - public RelNode convertSelect(SqlSelect select) { - final SqlValidatorScope selectScope = validator.getWhereScope(select); - final Blackboard bb = createBlackboard(selectScope, null); - convertSelectImpl(bb, select); - return bb.root; + Type elementType; + if (operator instanceof SqlUserDefinedTableFunction) { + SqlUserDefinedTableFunction udtf = (SqlUserDefinedTableFunction) operator; + elementType = udtf.getElementType(typeFactory, callBinding.operands()); + } else { + elementType = null; } - /** - * Factory method for creating translation workspace. - */ - protected Blackboard createBlackboard(SqlValidatorScope scope, Map nameToNodeMap) { - return new Blackboard(scope, nameToNodeMap); + RexNode rexCall = bb.convertExpression(call); + final List inputs = bb.retrieveCursors(); + Set columnMappings = + getColumnMappings(operator); + LogicalTableFunctionScan callRel = + LogicalTableFunctionScan.create( + cluster, + inputs, + rexCall, + elementType, + validator.getValidatedNodeType(call), + columnMappings); + bb.setRoot(callRel, true); + afterTableFunction(bb, call, callRel); + } + + protected void afterTableFunction( + SqlToRelConverter.Blackboard bb, + SqlCall call, + LogicalTableFunctionScan callRel) { + } + + private Set getColumnMappings(SqlOperator op) { + SqlReturnTypeInference rti = op.getReturnTypeInference(); + if (rti == null) { + return null; + } + if (rti instanceof TableFunctionReturnTypeInference) { + TableFunctionReturnTypeInference tfrti = + (TableFunctionReturnTypeInference) rti; + return tfrti.getColumnMappings(); + } else { + return null; + } + } + + protected RelNode createJoin( + Blackboard bb, + RelNode leftRel, + RelNode rightRel, + RexNode joinCond, + JoinRelType joinType) { + assert joinCond != null; + + final CorrelationUse p = getCorrelationUse(bb, rightRel); + if (p != null) { + LogicalCorrelate corr = LogicalCorrelate.create(leftRel, p.r, + p.id, p.requiredColumns, SemiJoinType.of(joinType)); + if (!joinCond.isAlwaysTrue()) { + return RelOptUtil.createFilter(corr, joinCond); + } + return corr; } - /** - * Implementation of {@link #convertSelect(SqlSelect)}; derived class may - * override. - */ - protected void convertSelectImpl(final Blackboard bb, SqlSelect select) { - convertFrom(bb, select.getFrom()); - convertWhere(bb, select.getWhere()); + final Join originalJoin = + (Join) RelFactories.DEFAULT_JOIN_FACTORY.createJoin(leftRel, rightRel, + joinCond, ImmutableSet.of(), joinType, false); - final List orderExprList = new ArrayList<>(); - final List collationList = new ArrayList<>(); - gatherOrderExprs(bb, select, select.getOrderList(), orderExprList, collationList); - final RelCollation collation = cluster.traitSet().canonize(RelCollations.of(collationList)); + return RelOptUtil.pushDownJoinConditions(originalJoin); + } - if (validator.isAggregate(select)) { - convertAgg(bb, select, orderExprList); + private CorrelationUse getCorrelationUse(Blackboard bb, final RelNode r0) { + final Set correlatedVariables = + RelOptUtil.getVariablesUsed(r0); + if (correlatedVariables.isEmpty()) { + return null; + } + final ImmutableBitSet.Builder requiredColumns = ImmutableBitSet.builder(); + final List correlNames = Lists.newArrayList(); + + // All correlations must refer the same namespace since correlation + // produces exactly one correlation source. + // The same source might be referenced by different variables since + // DeferredLookups are not de-duplicated at create time. + SqlValidatorNamespace prevNs = null; + + for (CorrelationId correlName : correlatedVariables) { + DeferredLookup lookup = + mapCorrelToDeferred.get(correlName); + RexFieldAccess fieldAccess = lookup.getFieldAccess(correlName); + String originalRelName = lookup.getOriginalRelName(); + String originalFieldName = fieldAccess.getField().getName(); + + int[] nsIndexes = {-1}; + final SqlValidatorScope[] ancestorScopes = {null}; + SqlValidatorNamespace foundNs = + lookup.bb.scope.resolve( + ImmutableList.of(originalRelName), + ancestorScopes, + nsIndexes); + + assert foundNs != null; + assert nsIndexes.length == 1; + + int childNamespaceIndex = nsIndexes[0]; + + SqlValidatorScope ancestorScope = ancestorScopes[0]; + boolean correlInCurrentScope = ancestorScope == bb.scope; + + if (!correlInCurrentScope) { + continue; + } + + if (prevNs == null) { + prevNs = foundNs; + } else { + assert prevNs == foundNs : "All correlation variables should resolve" + + " to the same namespace." + + " Prev ns=" + prevNs + + ", new ns=" + foundNs; + } + + int namespaceOffset = 0; + if (childNamespaceIndex > 0) { + // If not the first child, need to figure out the width + // of output types from all the preceding namespaces + assert ancestorScope instanceof ListScope; + List children = + ((ListScope) ancestorScope).getChildren(); + + for (int i = 0; i < childNamespaceIndex; i++) { + SqlValidatorNamespace child = children.get(i); + namespaceOffset += + child.getRowType().getFieldCount(); + } + } + + RelDataTypeField field = + catalogReader.field(foundNs.getRowType(), originalFieldName); + int pos = namespaceOffset + field.getIndex(); + + assert field.getType() + == lookup.getFieldAccess(correlName).getField().getType(); + + assert pos != -1; + + if (bb.mapRootRelToFieldProjection.containsKey(bb.root)) { + // bb.root is an aggregate and only projects group by + // keys. + Map exprProjection = + bb.mapRootRelToFieldProjection.get(bb.root); + + // subquery can reference group by keys projected from + // the root of the outer relation. + if (exprProjection.containsKey(pos)) { + pos = exprProjection.get(pos); } else { - convertSelectList(bb, select, orderExprList); + // correl not grouped + throw new AssertionError("Identifier '" + originalRelName + "." + + originalFieldName + "' is not a group expr"); } + } + + requiredColumns.set(pos); + correlNames.add(correlName); + } + + if (correlNames.isEmpty()) { + // None of the correlating variables originated in this scope. + return null; + } - if (select.isDistinct()) { - distinctify(bb, true); + RelNode r = r0; + if (correlNames.size() > 1) { + // The same table was referenced more than once. + // So we deduplicate + r = DeduplicateCorrelateVariables.go(rexBuilder, correlNames.get(0), + Util.skip(correlNames), r0); + } + return new CorrelationUse(correlNames.get(0), requiredColumns.build(), r); + } + + /** + * Determines whether a subquery is non-correlated. Note that a + * non-correlated subquery can contain correlated references, provided those + * references do not reference select statements that are parents of the + * subquery. + * + * @param subq the subquery + * @param bb blackboard used while converting the subquery, i.e., the + * blackboard of the parent query of this subquery + * @return true if the subquery is non-correlated. + */ + private boolean isSubQueryNonCorrelated(RelNode subq, Blackboard bb) { + Set correlatedVariables = RelOptUtil.getVariablesUsed(subq); + for (CorrelationId correlName : correlatedVariables) { + DeferredLookup lookup = mapCorrelToDeferred.get(correlName); + String originalRelName = lookup.getOriginalRelName(); + + int[] nsIndexes = {-1}; + final SqlValidatorScope[] ancestorScopes = {null}; + SqlValidatorNamespace foundNs = + lookup.bb.scope.resolve( + ImmutableList.of(originalRelName), + ancestorScopes, + nsIndexes); + + assert foundNs != null; + assert nsIndexes.length == 1; + + SqlValidatorScope ancestorScope = ancestorScopes[0]; + + // If the correlated reference is in a scope that's "above" the + // subquery, then this is a correlated subquery. + SqlValidatorScope parentScope = bb.scope; + do { + if (ancestorScope == parentScope) { + return false; + } + if (parentScope instanceof DelegatingScope) { + parentScope = ((DelegatingScope) parentScope).getParent(); + } else { + break; } - convertOrder(select, bb, collation, orderExprList, select.getOffset(), select.getFetch()); - bb.setRoot(bb.root, true); + } while (parentScope != null); + } + return true; + } + + /** + * Returns a list of fields to be prefixed to each relational expression. + * + * @return List of system fields + */ + protected List getSystemFields() { + return Collections.emptyList(); + } + + private RexNode convertJoinCondition(Blackboard bb, + SqlValidatorNamespace leftNamespace, + SqlValidatorNamespace rightNamespace, + SqlNode condition, + JoinConditionType conditionType, + RelNode leftRel, + RelNode rightRel) { + if (condition == null) { + return rexBuilder.makeLiteral(true); + } + bb.setRoot(ImmutableList.of(leftRel, rightRel)); + replaceSubqueries(bb, condition, RelOptUtil.Logic.UNKNOWN_AS_FALSE); + switch (conditionType) { + case ON: + bb.setRoot(ImmutableList.of(leftRel, rightRel)); + return bb.convertExpression(condition); + case USING: + final SqlNodeList list = (SqlNodeList) condition; + final List nameList = new ArrayList<>(); + for (SqlNode columnName : list) { + final SqlIdentifier id = (SqlIdentifier) columnName; + String name = id.getSimple(); + nameList.add(name); + } + return convertUsing(leftNamespace, rightNamespace, nameList); + default: + throw Util.unexpected(conditionType); + } + } + + /** + * Returns an expression for matching columns of a USING clause or inferred + * from NATURAL JOIN. "a JOIN b USING (x, y)" becomes "a.x = b.x AND a.y = + * b.y". Returns null if the column list is empty. + * + * @param leftNamespace Namespace of left input to join + * @param rightNamespace Namespace of right input to join + * @param nameList List of column names to join on + * @return Expression to match columns from name list, or true if name list + * is empty + */ + private RexNode convertUsing(SqlValidatorNamespace leftNamespace, + SqlValidatorNamespace rightNamespace, + List nameList) { + final List list = Lists.newArrayList(); + for (String name : nameList) { + List operands = new ArrayList<>(); + int offset = 0; + for (SqlValidatorNamespace n : ImmutableList.of(leftNamespace, + rightNamespace)) { + final RelDataType rowType = n.getRowType(); + final RelDataTypeField field = catalogReader.field(rowType, name); + operands.add( + rexBuilder.makeInputRef(field.getType(), + offset + field.getIndex())); + offset += rowType.getFieldList().size(); + } + list.add(rexBuilder.makeCall(SqlStdOperatorTable.EQUALS, operands)); + } + return RexUtil.composeConjunction(rexBuilder, list, false); + } + + private static JoinRelType convertJoinType(JoinType joinType) { + switch (joinType) { + case COMMA: + case INNER: + case CROSS: + return JoinRelType.INNER; + case FULL: + return JoinRelType.FULL; + case LEFT: + return JoinRelType.LEFT; + case RIGHT: + return JoinRelType.RIGHT; + default: + throw Util.unexpected(joinType); + } + } + + /** + * Converts the SELECT, GROUP BY and HAVING clauses of an aggregate query. + * + *

This method extracts SELECT, GROUP BY and HAVING clauses, and creates + * an {@link AggConverter}, then delegates to {@link #createAggImpl}. + * Derived class may override this method to change any of those clauses or + * specify a different {@link AggConverter}. + * + * @param bb Scope within which to resolve identifiers + * @param select Query + * @param orderExprList Additional expressions needed to implement ORDER BY + */ + protected void convertAgg( + Blackboard bb, + SqlSelect select, + List orderExprList) { + assert bb.root != null : "precondition: child != null"; + SqlNodeList groupList = select.getGroup(); + SqlNodeList selectList = select.getSelectList(); + SqlNode having = select.getHaving(); + + final AggConverter aggConverter = new AggConverter(bb, select); + createAggImpl( + bb, + aggConverter, + selectList, + groupList, + having, + orderExprList); + } + + protected final void createAggImpl( + Blackboard bb, + final AggConverter aggConverter, + SqlNodeList selectList, + SqlNodeList groupList, + SqlNode having, + List orderExprList) { + // Find aggregate functions in SELECT and HAVING clause + final AggregateFinder aggregateFinder = new AggregateFinder(); + selectList.accept(aggregateFinder); + if (having != null) { + having.accept(aggregateFinder); } - /** - * Having translated 'SELECT ... FROM ... [GROUP BY ...] [HAVING ...]', adds - * a relational expression to make the results unique. - * - *

If the SELECT clause contains duplicate expressions, adds - * {@link org.apache.calcite.rel.logical.LogicalProject}s so that we are - * grouping on the minimal set of keys. The performance gain isn't huge, but - * it is difficult to detect these duplicate expressions later. - * - * @param bb Blackboard - * @param checkForDupExprs Check for duplicate expressions - */ - private void distinctify(Blackboard bb, boolean checkForDupExprs) { - // Look for duplicate expressions in the project. - // Say we have 'select x, y, x, z'. - // Then dups will be {[2, 0]} - // and oldToNew will be {[0, 0], [1, 1], [2, 0], [3, 2]} - RelNode rel = bb.root; - if (checkForDupExprs && (rel instanceof LogicalProject)) { - LogicalProject project = (LogicalProject) rel; - final List projectExprs = project.getProjects(); - final List origins = new ArrayList<>(); - int dupCount = 0; - for (int i = 0; i < projectExprs.size(); i++) { - int x = findExpr(projectExprs.get(i), projectExprs, i); - if (x >= 0) { - origins.add(x); - ++dupCount; - } else { - origins.add(i); - } - } - if (dupCount == 0) { - distinctify(bb, false); - return; - } + // first replace the subqueries inside the aggregates + // because they will provide input rows to the aggregates. + replaceSubqueries(bb, aggregateFinder.list, RelOptUtil.Logic.TRUE_FALSE_UNKNOWN); - final Map squished = Maps.newHashMap(); - final List fields = rel.getRowType().getFieldList(); - final List> newProjects = Lists.newArrayList(); - for (int i = 0; i < fields.size(); i++) { - if (origins.get(i) == i) { - squished.put(i, newProjects.size()); - newProjects.add(RexInputRef.of2(i, fields)); - } - } - rel = LogicalProject.create(rel, Pair.left(newProjects), Pair.right(newProjects)); - bb.root = rel; - distinctify(bb, false); - rel = bb.root; - - // Create the expressions to reverse the mapping. - // Project($0, $1, $0, $2). - final List> undoProjects = Lists.newArrayList(); - for (int i = 0; i < fields.size(); i++) { - final int origin = origins.get(i); - RelDataTypeField field = fields.get(i); - undoProjects.add(Pair.of((RexNode) new RexInputRef(squished.get(origin), field.getType()), field.getName())); - } + // If group-by clause is missing, pretend that it has zero elements. + if (groupList == null) { + groupList = SqlNodeList.EMPTY; + } - rel = LogicalProject.create(rel, Pair.left(undoProjects), Pair.right(undoProjects)); - bb.setRoot(rel, false); + replaceSubqueries(bb, groupList, RelOptUtil.Logic.TRUE_FALSE_UNKNOWN); - return; - } + // register the group exprs - // Usual case: all of the expressions in the SELECT clause are - // different. - final ImmutableBitSet groupSet = ImmutableBitSet.range(rel.getRowType().getFieldCount()); - rel = createAggregate(bb, false, groupSet, ImmutableList.of(groupSet), ImmutableList. of()); + // build a map to remember the projections from the top scope to the + // output of the current root. + // + // Calcite allows expressions, not just column references in + // group by list. This is not SQL 2003 compliant, but hey. - bb.setRoot(rel, false); + final AggregatingSelectScope scope = aggConverter.aggregatingSelectScope; + final AggregatingSelectScope.Resolved r = scope.resolved.get(); + for (SqlNode groupExpr : r.groupExprList) { + aggConverter.addGroupExpr(groupExpr); } - private int findExpr(RexNode seek, List exprs, int count) { - for (int i = 0; i < count; i++) { - RexNode expr = exprs.get(i); - if (expr.toString().equals(seek.toString())) { - return i; - } - } - return -1; + RexNode havingExpr = null; + final List> projects = Lists.newArrayList(); + + try { + Util.permAssert(bb.agg == null, "already in agg mode"); + bb.agg = aggConverter; + + // convert the select and having expressions, so that the + // agg converter knows which aggregations are required + + selectList.accept(aggConverter); + for (SqlNode expr : orderExprList) { + expr.accept(aggConverter); + } + if (having != null) { + having.accept(aggConverter); + } + + // compute inputs to the aggregator + List preExprs = aggConverter.getPreExprs(); + List preNames = aggConverter.getPreNames(); + + if (preExprs.size() == 0) { + // Special case for COUNT(*), where we can end up with no inputs + // at all. The rest of the system doesn't like 0-tuples, so we + // select a dummy constant here. + preExprs = + ImmutableList.of( + rexBuilder.makeExactLiteral(BigDecimal.ZERO)); + preNames = Collections.singletonList(null); + } + + final RelNode inputRel = bb.root; + + // Project the expressions required by agg and having. + bb.setRoot( + RelOptUtil.createProject( + inputRel, + preExprs, + preNames, + true), + false); + bb.mapRootRelToFieldProjection.put(bb.root, r.groupExprProjection); + + // REVIEW jvs 31-Oct-2007: doesn't the declaration of + // monotonicity here assume sort-based aggregation at + // the physical level? + + // Tell bb which of group columns are sorted. + bb.columnMonotonicities.clear(); + for (SqlNode groupItem : groupList) { + bb.columnMonotonicities.add( + bb.scope.getMonotonicity(groupItem)); + } + + // Add the aggregator + bb.setRoot( + createAggregate(bb, r.indicator, r.groupSet, r.groupSets, + aggConverter.getAggCalls()), + false); + + // Generate NULL values for rolled-up not-null fields. + final Aggregate aggregate = (Aggregate) bb.root; + if (aggregate.getGroupType() != Aggregate.Group.SIMPLE) { + assert aggregate.indicator; + List> projects2 = Lists.newArrayList(); + int converted = 0; + final int groupCount = aggregate.getGroupSet().cardinality(); + for (RelDataTypeField field : aggregate.getRowType().getFieldList()) { + final int i = field.getIndex(); + final RexNode rex; + if (i < groupCount && r.isNullable(i)) { + ++converted; + + rex = rexBuilder.makeCall(SqlStdOperatorTable.CASE, + rexBuilder.makeInputRef(aggregate, groupCount + i), + rexBuilder.makeCast( + typeFactory.createTypeWithNullability( + field.getType(), true), + rexBuilder.constantNull()), + rexBuilder.makeInputRef(aggregate, i)); + } else { + rex = rexBuilder.makeInputRef(aggregate, i); + } + projects2.add(Pair.of(rex, field.getName())); + } + if (converted > 0) { + bb.setRoot( + RelOptUtil.createProject(bb.root, projects2, true), + false); + } + } + + bb.mapRootRelToFieldProjection.put(bb.root, r.groupExprProjection); + + // Replace subqueries in having here and modify having to use + // the replaced expressions + if (having != null) { + SqlNode newHaving = pushDownNotForIn(having); + replaceSubqueries(bb, newHaving, RelOptUtil.Logic.UNKNOWN_AS_FALSE); + havingExpr = bb.convertExpression(newHaving); + if (havingExpr.isAlwaysTrue()) { + havingExpr = null; + } + } + + // Now convert the other subqueries in the select list. + // This needs to be done separately from the subquery inside + // any aggregate in the select list, and after the aggregate rel + // is allocated. + replaceSubqueries(bb, selectList, RelOptUtil.Logic.TRUE_FALSE_UNKNOWN); + + // Now subqueries in the entire select list have been converted. + // Convert the select expressions to get the final list to be + // projected. + int k = 0; + + // For select expressions, use the field names previously assigned + // by the validator. If we derive afresh, we might generate names + // like "EXPR$2" that don't match the names generated by the + // validator. This is especially the case when there are system + // fields; system fields appear in the relnode's rowtype but do not + // (yet) appear in the validator type. + final SelectScope selectScope = + SqlValidatorUtil.getEnclosingSelectScope(bb.scope); + assert selectScope != null; + final SqlValidatorNamespace selectNamespace = + validator.getNamespace(selectScope.getNode()); + final List names = + selectNamespace.getRowType().getFieldNames(); + int sysFieldCount = selectList.size() - names.size(); + for (SqlNode expr : selectList) { + projects.add( + Pair.of(bb.convertExpression(expr), + k < sysFieldCount + ? validator.deriveAlias(expr, k++) + : names.get(k++ - sysFieldCount))); + } + + for (SqlNode expr : orderExprList) { + projects.add( + Pair.of(bb.convertExpression(expr), + validator.deriveAlias(expr, k++))); + } + } finally { + bb.agg = null; } - /** - * Converts a query's ORDER BY clause, if any. - * - * @param select Query - * @param bb Blackboard - * @param collation Collation list - * @param orderExprList Method populates this list with orderBy expressions - * not present in selectList - * @param offset Expression for number of rows to discard before - * returning first row - * @param fetch Expression for number of rows to fetch - */ - protected void convertOrder(SqlSelect select, Blackboard bb, RelCollation collation, List orderExprList, SqlNode offset, SqlNode fetch) { - if (select.getOrderList() == null) { - assert collation.getFieldCollations().isEmpty(); - if (offset == null && fetch == null) { - return; - } - } - - // Create a sorter using the previously constructed collations. - bb.setRoot(LogicalSort.create(bb.root, collation, offset == null ? null : convertExpression(offset), fetch == null ? null : convertExpression(fetch)), false); - - // If extra expressions were added to the project list for sorting, - // add another project to remove them. - if (orderExprList.size() > 0) { - final List exprs = new ArrayList<>(); - final RelDataType rowType = bb.root.getRowType(); - final int fieldCount = rowType.getFieldCount() - orderExprList.size(); - for (int i = 0; i < fieldCount; i++) { - exprs.add(rexBuilder.makeInputRef(bb.root, i)); - } - bb.setRoot(new LogicalProject(cluster, cluster.traitSetOf(RelCollations.PRESERVE), bb.root, exprs, cluster.getTypeFactory().createStructType(rowType.getFieldList().subList(0, fieldCount))), false); - } + // implement HAVING (we have already checked that it is non-trivial) + if (havingExpr != null) { + bb.setRoot(RelOptUtil.createFilter(bb.root, havingExpr), false); } - /** - * Returns whether a given node contains a {@link SqlInOperator}. - * - * @param node a RexNode tree - */ - private static boolean containsInOperator(SqlNode node) { - try { - SqlVisitor visitor = new SqlBasicVisitor() { - public Void visit(SqlCall call) { - if (call.getOperator() instanceof SqlInOperator) { - throw new Util.FoundOne(call); - } - return super.visit(call); - } - }; - node.accept(visitor); - return false; - } catch (Util.FoundOne e) { - Util.swallow(e, null); - return true; - } + // implement the SELECT list + bb.setRoot( + RelOptUtil.createProject( + bb.root, + projects, + true), + false); + + // Tell bb which of group columns are sorted. + bb.columnMonotonicities.clear(); + for (SqlNode selectItem : selectList) { + bb.columnMonotonicities.add( + bb.scope.getMonotonicity(selectItem)); + } + } + + /** + * Creates an Aggregate. + * + *

In case the aggregate rel changes the order in which it projects + * fields, the groupExprProjection parameter is provided, and + * the implementation of this method may modify it. + * + *

The sortedCount parameter is the number of expressions + * known to be monotonic. These expressions must be on the leading edge of + * the grouping keys. The default implementation of this method ignores this + * parameter. + * + * @param bb Blackboard + * @param indicator Whether to output fields indicating grouping sets + * @param groupSet Bit set of ordinals of grouping columns + * @param groupSets Grouping sets + * @param aggCalls Array of calls to aggregate functions + * @return LogicalAggregate + */ + protected RelNode createAggregate(Blackboard bb, boolean indicator, + ImmutableBitSet groupSet, ImmutableList groupSets, + List aggCalls) { + return LogicalAggregate.create( + bb.root, indicator, groupSet, groupSets, aggCalls); + } + + public RexDynamicParam convertDynamicParam( + final SqlDynamicParam dynamicParam) { + // REVIEW jvs 8-Jan-2005: dynamic params may be encountered out of + // order. Should probably cross-check with the count from the parser + // at the end and make sure they all got filled in. Why doesn't List + // have a resize() method?!? Make this a utility. + while (dynamicParam.getIndex() >= dynamicParamSqlNodes.size()) { + dynamicParamSqlNodes.add(null); } - /** - * Push down all the NOT logical operators into any IN/NOT IN operators. - * - * @param sqlNode the root node from which to look for NOT operators - * @return the transformed SqlNode representation with NOT pushed down. - */ - private static SqlNode pushDownNotForIn(SqlNode sqlNode) { - if ((sqlNode instanceof SqlCall) && containsInOperator(sqlNode)) { - SqlCall sqlCall = (SqlCall) sqlNode; - if ((sqlCall.getOperator() == SqlStdOperatorTable.AND) || (sqlCall.getOperator() == SqlStdOperatorTable.OR)) { - SqlNode[] sqlOperands = ((SqlBasicCall) sqlCall).operands; - for (int i = 0; i < sqlOperands.length; i++) { - sqlOperands[i] = pushDownNotForIn(sqlOperands[i]); - } - return sqlNode; - } else if (sqlCall.getOperator() == SqlStdOperatorTable.NOT) { - SqlNode childNode = sqlCall.operand(0); - assert childNode instanceof SqlCall; - SqlBasicCall childSqlCall = (SqlBasicCall) childNode; - if (childSqlCall.getOperator() == SqlStdOperatorTable.AND) { - SqlNode[] andOperands = childSqlCall.getOperands(); - SqlNode[] orOperands = new SqlNode[andOperands.length]; - for (int i = 0; i < orOperands.length; i++) { - orOperands[i] = SqlStdOperatorTable.NOT.createCall(SqlParserPos.ZERO, andOperands[i]); - } - for (int i = 0; i < orOperands.length; i++) { - orOperands[i] = pushDownNotForIn(orOperands[i]); - } - return SqlStdOperatorTable.OR.createCall(SqlParserPos.ZERO, orOperands[0], orOperands[1]); - } else if (childSqlCall.getOperator() == SqlStdOperatorTable.OR) { - SqlNode[] orOperands = childSqlCall.getOperands(); - SqlNode[] andOperands = new SqlNode[orOperands.length]; - for (int i = 0; i < andOperands.length; i++) { - andOperands[i] = SqlStdOperatorTable.NOT.createCall(SqlParserPos.ZERO, orOperands[i]); - } - for (int i = 0; i < andOperands.length; i++) { - andOperands[i] = pushDownNotForIn(andOperands[i]); - } - return SqlStdOperatorTable.AND.createCall(SqlParserPos.ZERO, andOperands[0], andOperands[1]); - } else if (childSqlCall.getOperator() == SqlStdOperatorTable.NOT) { - SqlNode[] notOperands = childSqlCall.getOperands(); - assert notOperands.length == 1; - return pushDownNotForIn(notOperands[0]); - } else if (childSqlCall.getOperator() instanceof SqlInOperator) { - SqlNode[] inOperands = childSqlCall.getOperands(); - SqlInOperator inOp = (SqlInOperator) childSqlCall.getOperator(); - if (inOp.isNotIn()) { - return SqlStdOperatorTable.IN.createCall(SqlParserPos.ZERO, inOperands[0], inOperands[1]); - } else { - return SqlStdOperatorTable.NOT_IN.createCall(SqlParserPos.ZERO, inOperands[0], inOperands[1]); - } - } else { - // childSqlCall is "leaf" node in a logical expression tree - // (only considering AND, OR, NOT) - return sqlNode; - } - } else { - // sqlNode is "leaf" node in a logical expression tree - // (only considering AND, OR, NOT) - return sqlNode; - } - } else { - // tree rooted at sqlNode does not contain inOperator - return sqlNode; - } + dynamicParamSqlNodes.set( + dynamicParam.getIndex(), + dynamicParam); + return rexBuilder.makeDynamicParam( + getDynamicParamType(dynamicParam.getIndex()), + dynamicParam.getIndex()); + } + + /** + * Creates a list of collations required to implement the ORDER BY clause, + * if there is one. Populates extraOrderExprs with any sort + * expressions which are not in the select clause. + * + * @param bb Scope within which to resolve identifiers + * @param select Select clause. Never null, because we invent a + * dummy SELECT if ORDER BY is applied to a set + * operation (UNION etc.) + * @param orderList Order by clause, may be null + * @param extraOrderExprs Sort expressions which are not in the select + * clause (output) + * @param collationList List of collations (output) + */ + protected void gatherOrderExprs( + Blackboard bb, + SqlSelect select, + SqlNodeList orderList, + List extraOrderExprs, + List collationList) { + // TODO: add validation rules to SqlValidator also + assert bb.root != null : "precondition: child != null"; + assert select != null; + if (orderList == null) { + return; + } + for (SqlNode orderItem : orderList) { + collationList.add( + convertOrderItem( + select, + orderItem, + extraOrderExprs, + RelFieldCollation.Direction.ASCENDING, + RelFieldCollation.NullDirection.UNSPECIFIED)); + } + } + + protected RelFieldCollation convertOrderItem( + SqlSelect select, + SqlNode orderItem, List extraExprs, + RelFieldCollation.Direction direction, + RelFieldCollation.NullDirection nullDirection) { + assert select != null; + // Handle DESC keyword, e.g. 'select a, b from t order by a desc'. + switch (orderItem.getKind()) { + case DESCENDING: + return convertOrderItem( + select, + ((SqlCall) orderItem).operand(0), + extraExprs, + RelFieldCollation.Direction.DESCENDING, + nullDirection); + case NULLS_FIRST: + return convertOrderItem( + select, + ((SqlCall) orderItem).operand(0), + extraExprs, + direction, + RelFieldCollation.NullDirection.FIRST); + case NULLS_LAST: + return convertOrderItem( + select, + ((SqlCall) orderItem).operand(0), + extraExprs, + direction, + RelFieldCollation.NullDirection.LAST); } - /** - * Converts a WHERE clause. - * - * @param bb Blackboard - * @param where WHERE clause, may be null - */ - private void convertWhere(final Blackboard bb, final SqlNode where) { - if (where == null) { - return; - } - SqlNode newWhere = pushDownNotForIn(where); - replaceSubqueries(bb, newWhere, RelOptUtil.Logic.UNKNOWN_AS_FALSE); - final RexNode convertedWhere = bb.convertExpression(newWhere); + SqlNode converted = validator.expandOrderExpr(select, orderItem); - // only allocate filter if the condition is not TRUE - if (!convertedWhere.isAlwaysTrue()) { - bb.setRoot(RelOptUtil.createFilter(bb.root, convertedWhere), false); - } + switch (nullDirection) { + case UNSPECIFIED: + nullDirection = validator.getDefaultNullCollation().last(desc(direction)) + ? RelFieldCollation.NullDirection.LAST + : RelFieldCollation.NullDirection.FIRST; } - private void replaceSubqueries(final Blackboard bb, final SqlNode expr, RelOptUtil.Logic logic) { - findSubqueries(bb, expr, logic, false); - for (SubQuery node : bb.subqueryList) { - substituteSubquery(bb, node); - } + // Scan the select list and order exprs for an identical expression. + final SelectScope selectScope = validator.getRawSelectScope(select); + int ordinal = -1; + for (SqlNode selectItem : selectScope.getExpandedSelectList()) { + ++ordinal; + if (converted.equalsDeep(stripAs(selectItem), false)) { + return new RelFieldCollation( + ordinal, direction, nullDirection); + } } - private void substituteSubquery(Blackboard bb, SubQuery subQuery) { - final RexNode expr = subQuery.expr; - if (expr != null) { - // Already done. - return; - } + for (SqlNode extraExpr : extraExprs) { + ++ordinal; + if (converted.equalsDeep(extraExpr, false)) { + return new RelFieldCollation( + ordinal, direction, nullDirection); + } + } - final SqlBasicCall call; - final RelNode rel; - final SqlNode query; - final Pair converted; - switch (subQuery.node.getKind()) { - case CURSOR: - convertCursor(bb, subQuery); - return; - - case MULTISET_QUERY_CONSTRUCTOR: - case MULTISET_VALUE_CONSTRUCTOR: - rel = convertMultisets(ImmutableList.of(subQuery.node), bb); - subQuery.expr = bb.register(rel, JoinRelType.INNER); - return; + // TODO: handle collation sequence + // TODO: flag expressions as non-standard - case IN: - call = (SqlBasicCall) subQuery.node; - final SqlNode[] operands = call.getOperands(); + extraExprs.add(converted); + return new RelFieldCollation(ordinal + 1, direction, nullDirection); + } - SqlNode leftKeyNode = operands[0]; - query = operands[1]; + private static boolean desc(RelFieldCollation.Direction direction) { + switch (direction) { + case DESCENDING: + case STRICTLY_DESCENDING: + return true; + default: + return false; + } + } + + protected boolean enableDecorrelation() { + // disable subquery decorrelation when needed. + // e.g. if outer joins are not supported. + return decorrelationEnabled; + } + + protected RelNode decorrelateQuery(RelNode rootRel) { + return RelDecorrelator.decorrelateQuery(rootRel); + } + + /** + * Sets whether to trim unused fields as part of the conversion process. + * + * @param trim Whether to trim unused fields + */ + public void setTrimUnusedFields(boolean trim) { + this.trimUnusedFields = trim; + } + + /** + * Returns whether to trim unused fields as part of the conversion process. + * + * @return Whether to trim unused fields + */ + public boolean isTrimUnusedFields() { + /* OVERRIDE POINT */ + return false; + } + + public void setExpand(boolean expand) { + this.expand = expand; + } + + /** + * Recursively converts a query to a relational expression. + * + * @param query Query + * @param top Whether this query is the top-level query of the + * statement + * @param targetRowType Target row type, or null + * @return Relational expression + */ + protected RelRoot convertQueryRecursive(SqlNode query, boolean top, + RelDataType targetRowType) { + final SqlKind kind = query.getKind(); + switch (kind) { + case SELECT: + return RelRoot.of(convertSelect((SqlSelect) query, top), kind); + case INSERT: + return RelRoot.of(convertInsert((SqlInsert) query), kind); + case DELETE: + return RelRoot.of(convertDelete((SqlDelete) query), kind); + case UPDATE: + return RelRoot.of(convertUpdate((SqlUpdate) query), kind); + case MERGE: + return RelRoot.of(convertMerge((SqlMerge) query), kind); + case UNION: + case INTERSECT: + case EXCEPT: + return RelRoot.of(convertSetOp((SqlCall) query), kind); + case WITH: + return convertWith((SqlWith) query, top); + case VALUES: + return RelRoot.of(convertValues((SqlCall) query, targetRowType), kind); + default: + throw Util.newInternal("not a query: " + query); + } + } + + /** + * Converts a set operation (UNION, INTERSECT, MINUS) into relational + * expressions. + * + * @param call Call to set operator + * @return Relational expression + */ + protected RelNode convertSetOp(SqlCall call) { + final RelNode left = + convertQueryRecursive(call.operand(0), false, null).project(); + final RelNode right = + convertQueryRecursive(call.operand(1), false, null).project(); + boolean all = false; + if (call.getOperator() instanceof SqlSetOperator) { + all = ((SqlSetOperator) (call.getOperator())).isAll(); + } + switch (call.getKind()) { + case UNION: + return LogicalUnion.create(ImmutableList.of(left, right), all); + + case INTERSECT: + // TODO: all + if (!all) { + return LogicalIntersect.create(ImmutableList.of(left, right), all); + } else { + throw Util.newInternal( + "set operator INTERSECT ALL not suported"); + } + + case EXCEPT: + // TODO: all + if (!all) { + return LogicalMinus.create(ImmutableList.of(left, right), all); + } else { + throw Util.newInternal( + "set operator EXCEPT ALL not suported"); + } + + default: + throw Util.unexpected(call.getKind()); + } + } + + protected RelNode convertInsert(SqlInsert call) { + RelOptTable targetTable = getTargetTable(call); + + final RelDataType targetRowType = + validator.getValidatedNodeType(call); + assert targetRowType != null; + RelNode sourceRel = + convertQueryRecursive(call.getSource(), false, targetRowType).project(); + RelNode massagedRel = convertColumnList(call, sourceRel); + + return createModify(targetTable, massagedRel); + } + + /** Creates a relational expression to modify a table or modifiable view. */ + private RelNode createModify(RelOptTable targetTable, RelNode source) { + final ModifiableTable modifiableTable = + targetTable.unwrap(ModifiableTable.class); + if (modifiableTable != null) { + return modifiableTable.toModificationRel(cluster, targetTable, + catalogReader, source, LogicalTableModify.Operation.INSERT, null, + false); + } + final ModifiableView modifiableView = + targetTable.unwrap(ModifiableView.class); + if (modifiableView != null) { + final Table delegateTable = modifiableView.getTable(); + final RelDataType delegateRowType = delegateTable.getRowType(typeFactory); + final RelOptTable delegateRelOptTable = + RelOptTableImpl.create(null, delegateRowType, delegateTable, + modifiableView.getTablePath()); + final RelNode newSource = + createSource(targetTable, source, modifiableView, delegateRowType); + return createModify(delegateRelOptTable, newSource); + } + return LogicalTableModify.create(targetTable, catalogReader, source, + LogicalTableModify.Operation.INSERT, null, false); + } + + /** Wraps a relational expression in the projects and filters implied by + * a {@link ModifiableView}. + * + *

The input relational expression is suitable for inserting into the view, + * and the returned relational expression is suitable for inserting into its + * delegate table. + * + *

In principle, the delegate table of a view might be another modifiable + * view, and if so, the process can be repeated. */ + private RelNode createSource(RelOptTable targetTable, RelNode source, + ModifiableView modifiableView, RelDataType delegateRowType) { + final ImmutableIntList mapping = modifiableView.getColumnMapping(); + assert mapping.size() == targetTable.getRowType().getFieldCount(); + + // For columns represented in the mapping, the expression is just a field + // reference. + final Map projectMap = new HashMap<>(); + final List filters = new ArrayList<>(); + for (int i = 0; i < mapping.size(); i++) { + int target = mapping.get(i); + if (target >= 0) { + projectMap.put(target, RexInputRef.of(i, source.getRowType())); + } + } - final List leftKeys; - switch (leftKeyNode.getKind()) { - case ROW: - leftKeys = Lists.newArrayList(); - for (SqlNode sqlExpr : ((SqlBasicCall) leftKeyNode).getOperandList()) { - leftKeys.add(bb.convertExpression(sqlExpr)); - } - break; - default: - leftKeys = ImmutableList.of(bb.convertExpression(leftKeyNode)); - } + // For columns that are not in the mapping, and have a constraint of the + // form "column = value", the expression is the literal "value". + // + // If a column has multiple constraints, the extra ones will become a + // filter. + final RexNode constraint = + modifiableView.getConstraint(rexBuilder, delegateRowType); + RelOptUtil.inferViewPredicates(projectMap, filters, constraint); + final List> projects = new ArrayList<>(); + for (RelDataTypeField field : delegateRowType.getFieldList()) { + RexNode node = projectMap.get(field.getIndex()); + if (node == null) { + node = rexBuilder.makeNullLiteral(field.getType().getSqlTypeName()); + } + projects.add( + Pair.of(rexBuilder.ensureType(field.getType(), node, false), + field.getName())); + } - final boolean isNotIn = ((SqlInOperator) call.getOperator()).isNotIn(); - if (query instanceof SqlNodeList) { - SqlNodeList valueList = (SqlNodeList) query; - if (!containsNullLiteral(valueList) && valueList.size() < getInSubqueryThreshold()) { - // We're under the threshold, so convert to OR. - subQuery.expr = convertInToOr(bb, leftKeys, valueList, isNotIn); - return; - } + source = RelOptUtil.createProject(source, projects, true); + if (filters.size() > 0) { + source = RelOptUtil.createFilter(source, filters); + } + return source; + } - // Otherwise, let convertExists translate - // values list into an inline table for the - // reference to Q below. - } + private RelOptTable.ToRelContext createToRelContext() { + return new RelOptTable.ToRelContext() { + public RelOptCluster getCluster() { + return cluster; + } + + public RelRoot expandView(RelDataType rowType, String queryString, + List schemaPath) { + return viewExpander.expandView(rowType, queryString, schemaPath); + } + }; + } + + public RelNode toRel(RelOptTable table) { + return table.toRel(createToRelContext()); + } + + protected RelOptTable getTargetTable(SqlNode call) { + SqlValidatorNamespace targetNs = validator.getNamespace(call).resolve(); + return SqlValidatorUtil.getRelOptTable(targetNs, catalogReader, null, null); + } + + /** + * Creates a source for an INSERT statement. + * + *

If the column list is not specified, source expressions match target + * columns in order. + * + *

If the column list is specified, Source expressions are mapped to + * target columns by name via targetColumnList, and may not cover the entire + * target table. So, we'll make up a full row, using a combination of + * default values and the source expressions provided. + * + * @param call Insert expression + * @param sourceRel Source relational expression + * @return Converted INSERT statement + */ + protected RelNode convertColumnList( + SqlInsert call, + RelNode sourceRel) { + RelDataType sourceRowType = sourceRel.getRowType(); + final RexNode sourceRef = + rexBuilder.makeRangeReference(sourceRowType, 0, false); + final List targetColumnNames = new ArrayList<>(); + final List columnExprs = new ArrayList<>(); + collectInsertTargets(call, sourceRef, targetColumnNames, columnExprs); + + final RelOptTable targetTable = getTargetTable(call); + final RelDataType targetRowType = targetTable.getRowType(); + final List targetFields = + targetRowType.getFieldList(); + final List sourceExps = + new ArrayList<>( + Collections.nCopies(targetFields.size(), null)); + final List fieldNames = + new ArrayList<>( + Collections.nCopies(targetFields.size(), null)); + + // Walk the name list and place the associated value in the + // expression list according to the ordinal value returned from + // the table construct, leaving nulls in the list for columns + // that are not referenced. + for (Pair p : Pair.zip(targetColumnNames, columnExprs)) { + RelDataTypeField field = catalogReader.field(targetRowType, p.left); + assert field != null : "column " + p.left + " not found"; + sourceExps.set(field.getIndex(), p.right); + } - // Project out the search columns from the left side - - // Q1: - // "select from emp where emp.deptno in (select col1 from T)" - // - // is converted to - // - // "select from - // emp inner join (select distinct col1 from T)) q - // on emp.deptno = q.col1 - // - // Q2: - // "select from emp where emp.deptno not in (Q)" - // - // is converted to - // - // "select from - // emp left outer join (select distinct col1, TRUE from T) q - // on emp.deptno = q.col1 - // where emp.deptno <> null - // and q.indicator <> TRUE" - // - final boolean outerJoin = bb.subqueryNeedsOuterJoin || isNotIn || subQuery.logic == RelOptUtil.Logic.TRUE_FALSE_UNKNOWN; - converted = convertExists(query, RelOptUtil.SubqueryType.IN, subQuery.logic, outerJoin); - if (converted.right) { - // Generate - // emp CROSS JOIN (SELECT COUNT(*) AS c, - // COUNT(deptno) AS ck FROM dept) - final RelDataType longType = typeFactory.createSqlType(SqlTypeName.BIGINT); - final RelNode seek = converted.left.getInput(0); // fragile - final int keyCount = leftKeys.size(); - final List args = ImmutableIntList.range(0, keyCount); - LogicalAggregate aggregate = LogicalAggregate.create(seek, false, ImmutableBitSet.of(), null, ImmutableList.of(AggregateCall.create(SqlStdOperatorTable.COUNT, false, ImmutableList. of(), -1, longType, null), AggregateCall.create(SqlStdOperatorTable.COUNT, false, args, -1, longType, null))); - LogicalJoin join = LogicalJoin.create(bb.root, aggregate, rexBuilder.makeLiteral(true), JoinRelType.INNER, ImmutableSet. of()); - bb.setRoot(join, false); - } - RexNode rex = bb.register(converted.left, outerJoin ? JoinRelType.LEFT : JoinRelType.INNER, leftKeys); + // Walk the expression list and get default values for any columns + // that were not supplied in the statement. Get field names too. + for (int i = 0; i < targetFields.size(); ++i) { + final RelDataTypeField field = targetFields.get(i); + final String fieldName = field.getName(); + fieldNames.set(i, fieldName); + if (sourceExps.get(i) != null) { + if (defaultValueFactory.isGeneratedAlways(targetTable, i)) { + throw RESOURCE.insertIntoAlwaysGenerated(fieldName).ex(); + } + continue; + } + sourceExps.set( + i, defaultValueFactory.newColumnDefaultValue(targetTable, i)); + + // bare nulls are dangerous in the wrong hands + sourceExps.set( + i, + castNullLiteralIfNeeded( + sourceExps.get(i), field.getType())); + } - subQuery.expr = translateIn(subQuery, bb.root, rex); - if (isNotIn) { - subQuery.expr = rexBuilder.makeCall(SqlStdOperatorTable.NOT, subQuery.expr); - } - return; + return RelOptUtil.createProject(sourceRel, sourceExps, fieldNames, true); + } - case EXISTS: - // "select from emp where exists (select a from T)" - // - // is converted to the following if the subquery is correlated: - // - // "select from emp left outer join (select AGG_TRUE() as indicator - // from T group by corr_var) q where q.indicator is true" - // - // If there is no correlation, the expression is replaced with a - // boolean indicating whether the subquery returned 0 or >= 1 row. - call = (SqlBasicCall) subQuery.node; - query = call.getOperands()[0]; - converted = convertExists(query, RelOptUtil.SubqueryType.EXISTS, subQuery.logic, true); - assert !converted.right; - if (convertNonCorrelatedSubQuery(subQuery, bb, converted.left, true)) { - return; - } - subQuery.expr = bb.register(converted.left, JoinRelType.LEFT); - return; + private RexNode castNullLiteralIfNeeded(RexNode node, RelDataType type) { + if (!RexLiteral.isNullLiteral(node)) { + return node; + } + return rexBuilder.makeCast(type, node); + } + + /** + * Given an INSERT statement, collects the list of names to be populated and + * the expressions to put in them. + * + * @param call Insert statement + * @param sourceRef Expression representing a row from the source + * relational expression + * @param targetColumnNames List of target column names, to be populated + * @param columnExprs List of expressions, to be populated + */ + protected void collectInsertTargets( + SqlInsert call, + final RexNode sourceRef, + final List targetColumnNames, + List columnExprs) { + final RelOptTable targetTable = getTargetTable(call); + final RelDataType targetRowType = targetTable.getRowType(); + SqlNodeList targetColumnList = call.getTargetColumnList(); + if (targetColumnList == null) { + targetColumnNames.addAll(targetRowType.getFieldNames()); + } else { + for (int i = 0; i < targetColumnList.size(); i++) { + SqlIdentifier id = (SqlIdentifier) targetColumnList.get(i); + targetColumnNames.add(id.getSimple()); + } + } - case SCALAR_QUERY: - // Convert the subquery. If it's non-correlated, convert it - // to a constant expression. - call = (SqlBasicCall) subQuery.node; - query = call.getOperands()[0]; - converted = convertExists(query, RelOptUtil.SubqueryType.SCALAR, subQuery.logic, true); - assert !converted.right; - if (convertNonCorrelatedSubQuery(subQuery, bb, converted.left, false)) { - return; - } - rel = convertToSingleValueSubq(query, converted.left); - subQuery.expr = bb.register(rel, JoinRelType.LEFT); - return; - - case SELECT: - // This is used when converting multiset queries: - // - // select * from unnest(select multiset[deptno] from emps); - // - converted = convertExists(subQuery.node, RelOptUtil.SubqueryType.SCALAR, subQuery.logic, true); - assert !converted.right; - subQuery.expr = bb.register(converted.left, JoinRelType.LEFT); - return; - - default: - throw Util.newInternal("unexpected kind of subquery :" + subQuery.node); - } + for (int i = 0; i < targetColumnNames.size(); i++) { + final RexNode expr = rexBuilder.makeFieldAccess(sourceRef, i); + columnExprs.add(expr); + } + } + + private RelNode convertDelete(SqlDelete call) { + RelOptTable targetTable = getTargetTable(call); + RelNode sourceRel = convertSelect(call.getSourceSelect(), false); + return LogicalTableModify.create(targetTable, catalogReader, sourceRel, + LogicalTableModify.Operation.DELETE, null, false); + } + + private RelNode convertUpdate(SqlUpdate call) { + RelOptTable targetTable = getTargetTable(call); + + // convert update column list from SqlIdentifier to String + final List targetColumnNameList = new ArrayList<>(); + for (SqlNode node : call.getTargetColumnList()) { + SqlIdentifier id = (SqlIdentifier) node; + String name = id.getSimple(); + targetColumnNameList.add(name); } - private RexNode translateIn(SubQuery subQuery, RelNode root, final RexNode rex) { - switch (subQuery.logic) { - case TRUE: - return rexBuilder.makeLiteral(true); - - case UNKNOWN_AS_FALSE: - assert rex instanceof RexRangeRef; - final int fieldCount = rex.getType().getFieldCount(); - RexNode rexNode = rexBuilder.makeFieldAccess(rex, fieldCount - 1); - rexNode = rexBuilder.makeCall(SqlStdOperatorTable.IS_TRUE, rexNode); - - // Then append the IS NOT NULL(leftKeysForIn). - // - // RexRangeRef contains the following fields: - // leftKeysForIn, - // rightKeysForIn (the original subquery select list), - // nullIndicator - // - // The first two lists contain the same number of fields. - final int k = (fieldCount - 1) / 2; - for (int i = 0; i < k; i++) { - rexNode = rexBuilder.makeCall(SqlStdOperatorTable.AND, rexNode, rexBuilder.makeCall(SqlStdOperatorTable.IS_NOT_NULL, rexBuilder.makeFieldAccess(rex, i))); - } - return rexNode; - - case TRUE_FALSE_UNKNOWN: - case UNKNOWN_AS_TRUE: - // select e.deptno, - // case - // when ct.c = 0 then false - // when dt.i is not null then true - // when e.deptno is null then null - // when ct.ck < ct.c then null - // else false - // end - // from e - // cross join (select count(*) as c, count(deptno) as ck from v) as ct - // left join (select distinct deptno, true as i from v) as dt - // on e.deptno = dt.deptno - final Join join = (Join) root; - final Project left = (Project) join.getLeft(); - final RelNode leftLeft = ((Join) left.getInput()).getLeft(); - final int leftLeftCount = leftLeft.getRowType().getFieldCount(); - final RelDataType nullableBooleanType = typeFactory.createTypeWithNullability(typeFactory.createSqlType(SqlTypeName.BOOLEAN), true); - final RelDataType longType = typeFactory.createSqlType(SqlTypeName.BIGINT); - final RexNode cRef = rexBuilder.makeInputRef(root, leftLeftCount); - final RexNode ckRef = rexBuilder.makeInputRef(root, leftLeftCount + 1); - final RexNode iRef = rexBuilder.makeInputRef(root, root.getRowType().getFieldCount() - 1); - - final RexLiteral zero = rexBuilder.makeExactLiteral(BigDecimal.ZERO, longType); - final RexLiteral trueLiteral = rexBuilder.makeLiteral(true); - final RexLiteral falseLiteral = rexBuilder.makeLiteral(false); - final RexNode unknownLiteral = rexBuilder.makeNullLiteral(SqlTypeName.BOOLEAN); - - final ImmutableList.Builder args = ImmutableList.builder(); - args.add(rexBuilder.makeCall(SqlStdOperatorTable.EQUALS, cRef, zero), falseLiteral, rexBuilder.makeCall(SqlStdOperatorTable.IS_NOT_NULL, iRef), trueLiteral); - final JoinInfo joinInfo = join.analyzeCondition(); - for (int leftKey : joinInfo.leftKeys) { - final RexNode kRef = rexBuilder.makeInputRef(root, leftKey); - args.add(rexBuilder.makeCall(SqlStdOperatorTable.IS_NULL, kRef), unknownLiteral); - } - args.add(rexBuilder.makeCall(SqlStdOperatorTable.LESS_THAN, ckRef, cRef), unknownLiteral, falseLiteral); + RelNode sourceRel = convertSelect(call.getSourceSelect(), false); - return rexBuilder.makeCall(nullableBooleanType, SqlStdOperatorTable.CASE, args.build()); + return LogicalTableModify.create(targetTable, catalogReader, sourceRel, + LogicalTableModify.Operation.UPDATE, targetColumnNameList, false); + } - default: - throw new AssertionError(subQuery.logic); - } + private RelNode convertMerge(SqlMerge call) { + RelOptTable targetTable = getTargetTable(call); + + // convert update column list from SqlIdentifier to String + final List targetColumnNameList = new ArrayList<>(); + SqlUpdate updateCall = call.getUpdateCall(); + if (updateCall != null) { + for (SqlNode targetColumn : updateCall.getTargetColumnList()) { + SqlIdentifier id = (SqlIdentifier) targetColumn; + String name = id.getSimple(); + targetColumnNameList.add(name); + } } - private static boolean containsNullLiteral(SqlNodeList valueList) { - for (SqlNode node : valueList.getList()) { - if (node instanceof SqlLiteral) { - SqlLiteral lit = (SqlLiteral) node; - if (lit.getValue() == null) { - return true; - } - } - } - return false; + // replace the projection of the source select with a + // projection that contains the following: + // 1) the expressions corresponding to the new insert row (if there is + // an insert) + // 2) all columns from the target table (if there is an update) + // 3) the set expressions in the update call (if there is an update) + + // first, convert the merge's source select to construct the columns + // from the target table and the set expressions in the update call + RelNode mergeSourceRel = convertSelect(call.getSourceSelect(), false); + + // then, convert the insert statement so we can get the insert + // values expressions + SqlInsert insertCall = call.getInsertCall(); + int nLevel1Exprs = 0; + List level1InsertExprs = null; + List level2InsertExprs = null; + if (insertCall != null) { + RelNode insertRel = convertInsert(insertCall); + + // if there are 2 level of projections in the insert source, combine + // them into a single project; level1 refers to the topmost project; + // the level1 projection contains references to the level2 + // expressions, except in the case where no target expression was + // provided, in which case, the expression is the default value for + // the column; or if the expressions directly map to the source + // table + level1InsertExprs = + ((LogicalProject) insertRel.getInput(0)).getProjects(); + if (insertRel.getInput(0).getInput(0) instanceof LogicalProject) { + level2InsertExprs = + ((LogicalProject) insertRel.getInput(0).getInput(0)) + .getProjects(); + } + nLevel1Exprs = level1InsertExprs.size(); } - /** - * Determines if a subquery is non-correlated and if so, converts it to a - * constant. - * - * @param subQuery the call that references the subquery - * @param bb blackboard used to convert the subquery - * @param converted RelNode tree corresponding to the subquery - * @param isExists true if the subquery is part of an EXISTS expression - * @return if the subquery can be converted to a constant - */ - private boolean convertNonCorrelatedSubQuery(SubQuery subQuery, Blackboard bb, RelNode converted, boolean isExists) { - SqlCall call = (SqlBasicCall) subQuery.node; - if (subqueryConverter.canConvertSubquery() && isSubQueryNonCorrelated(converted, bb)) { - // First check if the subquery has already been converted - // because it's a nested subquery. If so, don't re-evaluate - // it again. - RexNode constExpr = mapConvertedNonCorrSubqs.get(call); - if (constExpr == null) { - constExpr = subqueryConverter.convertSubquery(call, this, isExists, isExplain); - } - if (constExpr != null) { - subQuery.expr = constExpr; - mapConvertedNonCorrSubqs.put(call, constExpr); - return true; - } - } - return false; + LogicalJoin join = (LogicalJoin) mergeSourceRel.getInput(0); + int nSourceFields = join.getLeft().getRowType().getFieldCount(); + final List projects = new ArrayList<>(); + for (int level1Idx = 0; level1Idx < nLevel1Exprs; level1Idx++) { + if ((level2InsertExprs != null) + && (level1InsertExprs.get(level1Idx) instanceof RexInputRef)) { + int level2Idx = + ((RexInputRef) level1InsertExprs.get(level1Idx)).getIndex(); + projects.add(level2InsertExprs.get(level2Idx)); + } else { + projects.add(level1InsertExprs.get(level1Idx)); + } + } + if (updateCall != null) { + final LogicalProject project = (LogicalProject) mergeSourceRel; + projects.addAll( + Util.skip(project.getProjects(), nSourceFields)); } - /** - * Converts the RelNode tree for a select statement to a select that - * produces a single value. - * - * @param query the query - * @param plan the original RelNode tree corresponding to the statement - * @return the converted RelNode tree - */ - public RelNode convertToSingleValueSubq(SqlNode query, RelNode plan) { - // Check whether query is guaranteed to produce a single value. - if (query instanceof SqlSelect) { - SqlSelect select = (SqlSelect) query; - SqlNodeList selectList = select.getSelectList(); - SqlNodeList groupList = select.getGroup(); - - if ((selectList.size() == 1) && ((groupList == null) || (groupList.size() == 0))) { - SqlNode selectExpr = selectList.get(0); - if (selectExpr instanceof SqlCall) { - SqlCall selectExprCall = (SqlCall) selectExpr; - if (Util.isSingleValue(selectExprCall)) { - return plan; - } - } + RelNode massagedRel = + RelOptUtil.createProject(join, projects, null, true); + + return LogicalTableModify.create(targetTable, catalogReader, massagedRel, + LogicalTableModify.Operation.MERGE, targetColumnNameList, false); + } + + /** + * Converts an identifier into an expression in a given scope. For example, + * the "empno" in "select empno from emp join dept" becomes "emp.empno". + */ + private RexNode convertIdentifier( + Blackboard bb, + SqlIdentifier identifier) { + // first check for reserved identifiers like CURRENT_USER + final SqlCall call = SqlUtil.makeCall(opTab, identifier); + if (call != null) { + return bb.convertExpression(call); + } + + final SqlQualified qualified; + if (bb.scope != null) { + qualified = bb.scope.fullyQualify(identifier); + } else { + qualified = SqlQualified.create(null, 1, null, identifier); + } + final RexNode e0 = bb.lookupExp(qualified); + RexNode e = e0; + for (String name : qualified.suffixTranslated()) { + final boolean caseSensitive = true; // name already fully-qualified + e = rexBuilder.makeFieldAccess(e, name, caseSensitive); + } + if (e instanceof RexInputRef) { + // adjust the type to account for nulls introduced by outer joins + e = adjustInputRef(bb, (RexInputRef) e); + } - // If there is a limit with 0 or 1, - // it is ensured to produce a single value - if (select.getFetch() != null && select.getFetch() instanceof SqlNumericLiteral) { - SqlNumericLiteral limitNum = (SqlNumericLiteral) select.getFetch(); - if (((BigDecimal) limitNum.getValue()).intValue() < 2) { - return plan; - } + if (e0 instanceof RexCorrelVariable) { + assert e instanceof RexFieldAccess; + final RexNode prev = + bb.mapCorrelateToRex.put(((RexCorrelVariable) e0).id, + (RexFieldAccess) e); + assert prev == null; + } + return e; + } + + /** + * Adjusts the type of a reference to an input field to account for nulls + * introduced by outer joins; and adjusts the offset to match the physical + * implementation. + * + * @param bb Blackboard + * @param inputRef Input ref + * @return Adjusted input ref + */ + protected RexNode adjustInputRef( + Blackboard bb, + RexInputRef inputRef) { + RelDataTypeField field = bb.getRootField(inputRef); + if (field != null) { + return rexBuilder.makeInputRef( + field.getType(), + inputRef.getIndex()); + } + return inputRef; + } + + /** + * Converts a row constructor into a relational expression. + * + * @param bb Blackboard + * @param rowConstructor Row constructor expression + * @return Relational expression which returns a single row. + * @pre isRowConstructor(rowConstructor) + */ + private RelNode convertRowConstructor( + Blackboard bb, + SqlCall rowConstructor) { + assert isRowConstructor(rowConstructor) : rowConstructor; + final List operands = rowConstructor.getOperandList(); + return convertMultisets(operands, bb); + } + + private RelNode convertCursor(Blackboard bb, SubQuery subQuery) { + final SqlCall cursorCall = (SqlCall) subQuery.node; + assert cursorCall.operandCount() == 1; + SqlNode query = cursorCall.operand(0); + RelNode converted = convertQuery(query, false, false).rel; + int iCursor = bb.cursors.size(); + bb.cursors.add(converted); + subQuery.expr = + new RexInputRef( + iCursor, + converted.getRowType()); + return converted; + } + + private RelNode convertMultisets(final List operands, + Blackboard bb) { + // NOTE: Wael 2/04/05: this implementation is not the most efficient in + // terms of planning since it generates XOs that can be reduced. + final List joinList = new ArrayList<>(); + List lastList = new ArrayList<>(); + for (int i = 0; i < operands.size(); i++) { + SqlNode operand = operands.get(i); + if (!(operand instanceof SqlCall)) { + lastList.add(operand); + continue; + } + + final SqlCall call = (SqlCall) operand; + final RelNode input; + switch (call.getKind()) { + case MULTISET_VALUE_CONSTRUCTOR: + case ARRAY_VALUE_CONSTRUCTOR: + final SqlNodeList list = + new SqlNodeList(call.getOperandList(), call.getParserPosition()); + CollectNamespace nss = + (CollectNamespace) validator.getNamespace(call); + Blackboard usedBb; + if (null != nss) { + usedBb = createBlackboard(nss.getScope(), null, false); + } else { + usedBb = + createBlackboard(new ListScope(bb.scope) { + public SqlNode getNode() { + return call; } - } - } else if (query instanceof SqlCall) { - // If the query is (values ...), - // it is necessary to look into the operands to determine - // whether SingleValueAgg is necessary - SqlCall exprCall = (SqlCall) query; - if (exprCall.getOperator() instanceof SqlValuesOperator && Util.isSingleValue(exprCall)) { - return plan; - } - } + }, null, false); + } + RelDataType multisetType = validator.getValidatedNodeType(call); + validator.setValidatedNodeType( + list, + multisetType.getComponentType()); + input = convertQueryOrInList(usedBb, list, null); + break; + case MULTISET_QUERY_CONSTRUCTOR: + case ARRAY_QUERY_CONSTRUCTOR: + final RelRoot root = convertQuery(call.operand(0), false, true); + input = root.rel; + break; + default: + lastList.add(operand); + continue; + } + + if (lastList.size() > 0) { + joinList.add(lastList); + } + lastList = new ArrayList<>(); + Collect collect = + new Collect( + cluster, + cluster.traitSetOf(Convention.NONE), + input, + validator.deriveAlias(call, i)); + joinList.add(collect); + } - // If not, project SingleValueAgg - return RelOptUtil.createSingleValueAggRel(cluster, plan); + if (joinList.size() == 0) { + joinList.add(lastList); } - /** - * Converts "x IN (1, 2, ...)" to "x=1 OR x=2 OR ...". - * - * @param leftKeys LHS - * @param valuesList RHS - * @param isNotIn is this a NOT IN operator - * @return converted expression - */ - private RexNode convertInToOr(final Blackboard bb, final List leftKeys, SqlNodeList valuesList, boolean isNotIn) { - final List comparisons = new ArrayList<>(); - for (SqlNode rightVals : valuesList) { - RexNode rexComparison; - if (leftKeys.size() == 1) { - rexComparison = rexBuilder.makeCall(SqlStdOperatorTable.EQUALS, leftKeys.get(0), rexBuilder.ensureType(leftKeys.get(0).getType(), bb.convertExpression(rightVals), true)); - } else { - assert rightVals instanceof SqlCall; - final SqlBasicCall call = (SqlBasicCall) rightVals; - assert (call.getOperator() instanceof SqlRowOperator) && call.getOperands().length == leftKeys.size(); - rexComparison = RexUtil.composeConjunction(rexBuilder, Iterables.transform(Pair.zip(leftKeys, call.getOperandList()), new Function, RexNode>() { - public RexNode apply(Pair pair) { - return rexBuilder.makeCall(SqlStdOperatorTable.EQUALS, pair.left, rexBuilder.ensureType(pair.left.getType(), bb.convertExpression(pair.right), true)); - } - }), false); - } - comparisons.add(rexComparison); - } + for (int i = 0; i < joinList.size(); i++) { + Object o = joinList.get(i); + if (o instanceof List) { + @SuppressWarnings("unchecked") + List projectList = (List) o; + final List selectList = new ArrayList<>(); + final List fieldNameList = new ArrayList<>(); + for (int j = 0; j < projectList.size(); j++) { + SqlNode operand = projectList.get(j); + selectList.add(bb.convertExpression(operand)); + + // REVIEW angel 5-June-2005: Use deriveAliasFromOrdinal + // instead of deriveAlias to match field names from + // SqlRowOperator. Otherwise, get error Type + // 'RecordType(INTEGER EMPNO)' has no field 'EXPR$0' when + // doing select * from unnest( select multiset[empno] + // from sales.emps); + + fieldNameList.add(SqlUtil.deriveAliasFromOrdinal(j)); + } + + RelNode projRel = + RelOptUtil.createProject( + LogicalValues.createOneRow(cluster), + selectList, + fieldNameList); + + joinList.set(i, projRel); + } + } - RexNode result = RexUtil.composeDisjunction(rexBuilder, comparisons, true); - assert result != null; + RelNode ret = (RelNode) joinList.get(0); + for (int i = 1; i < joinList.size(); i++) { + RelNode relNode = (RelNode) joinList.get(i); + ret = + RelFactories.DEFAULT_JOIN_FACTORY.createJoin( + ret, + relNode, + rexBuilder.makeLiteral(true), + ImmutableSet.of(), + JoinRelType.INNER, + false); + } + return ret; + } + + private void convertSelectList( + Blackboard bb, + SqlSelect select, + List orderList) { + SqlNodeList selectList = select.getSelectList(); + selectList = validator.expandStar(selectList, select, false); + + replaceSubqueries(bb, selectList, RelOptUtil.Logic.TRUE_FALSE_UNKNOWN); + + List fieldNames = new ArrayList<>(); + final List exprs = new ArrayList<>(); + final Collection aliases = new TreeSet<>(); + + // Project any system fields. (Must be done before regular select items, + // because offsets may be affected.) + final List columnMonotonicityList = new ArrayList<>(); + extraSelectItems( + bb, + select, + exprs, + fieldNames, + aliases, + columnMonotonicityList); + + // Project select clause. + int i = -1; + for (SqlNode expr : selectList) { + ++i; + exprs.add(bb.convertExpression(expr)); + fieldNames.add(deriveAlias(expr, aliases, i)); + } - if (isNotIn) { - result = rexBuilder.makeCall(SqlStdOperatorTable.NOT, result); - } + // Project extra fields for sorting. + for (SqlNode expr : orderList) { + ++i; + SqlNode expr2 = validator.expandOrderExpr(select, expr); + exprs.add(bb.convertExpression(expr2)); + fieldNames.add(deriveAlias(expr, aliases, i)); + } - return result; + fieldNames = SqlValidatorUtil.uniquify(fieldNames); + + RelNode inputRel = bb.root; + bb.setRoot( + RelOptUtil.createProject(bb.root, exprs, fieldNames), + false); + + assert bb.columnMonotonicities.isEmpty(); + bb.columnMonotonicities.addAll(columnMonotonicityList); + for (SqlNode selectItem : selectList) { + bb.columnMonotonicities.add( + selectItem.getMonotonicity(bb.scope)); + } + } + + /** + * Adds extra select items. The default implementation adds nothing; derived + * classes may add columns to exprList, nameList, aliasList and + * columnMonotonicityList. + * + * @param bb Blackboard + * @param select Select statement being translated + * @param exprList List of expressions in select clause + * @param nameList List of names, one per column + * @param aliasList Collection of aliases that have been used + * already + * @param columnMonotonicityList List of monotonicity, one per column + */ + protected void extraSelectItems( + Blackboard bb, + SqlSelect select, + List exprList, + List nameList, + Collection aliasList, + List columnMonotonicityList) { + } + + private String deriveAlias( + final SqlNode node, + Collection aliases, + final int ordinal) { + String alias = validator.deriveAlias(node, ordinal); + if ((alias == null) || aliases.contains(alias)) { + String aliasBase = (alias == null) ? "EXPR$" : alias; + for (int j = 0;; j++) { + alias = aliasBase + j; + if (!aliases.contains(alias)) { + break; + } + } + } + aliases.add(alias); + return alias; + } + + /** + * Converts a WITH sub-query into a relational expression. + */ + public RelRoot convertWith(SqlWith with, boolean top) { + return convertQuery(with.body, false, top); + } + + /** + * Converts a SELECT statement's parse tree into a relational expression. + */ + public RelNode convertValues( + SqlCall values, + RelDataType targetRowType) { + final SqlValidatorScope scope = validator.getOverScope(values); + assert scope != null; + final Blackboard bb = createBlackboard(scope, null, false); + convertValuesImpl(bb, values, targetRowType); + return bb.root; + } + + /** + * Converts a values clause (as in "INSERT INTO T(x,y) VALUES (1,2)") into a + * relational expression. + * + * @param bb Blackboard + * @param values Call to SQL VALUES operator + * @param targetRowType Target row type + */ + private void convertValuesImpl( + Blackboard bb, + SqlCall values, + RelDataType targetRowType) { + // Attempt direct conversion to LogicalValues; if that fails, deal with + // fancy stuff like subqueries below. + RelNode valuesRel = + convertRowValues( + bb, + values, + values.getOperandList(), + true, + targetRowType); + if (valuesRel != null) { + bb.setRoot(valuesRel, true); + return; } - /** - * Gets the list size threshold under which {@link #convertInToOr} is used. - * Lists of this size or greater will instead be converted to use a join - * against an inline table - * ({@link org.apache.calcite.rel.logical.LogicalValues}) rather than a - * predicate. A threshold of 0 forces usage of an inline table in all cases; a - * threshold of Integer.MAX_VALUE forces usage of OR in all cases - * - * @return threshold, default 20 - */ - protected int getInSubqueryThreshold() { - /* OVERRIDE POINT */ - return Integer.MAX_VALUE; + final List unionRels = new ArrayList<>(); + for (SqlNode rowConstructor1 : values.getOperandList()) { + SqlCall rowConstructor = (SqlCall) rowConstructor1; + Blackboard tmpBb = createBlackboard(bb.scope, null, false); + replaceSubqueries(tmpBb, rowConstructor, + RelOptUtil.Logic.TRUE_FALSE_UNKNOWN); + final List> exps = new ArrayList<>(); + for (Ord operand : Ord.zip(rowConstructor.getOperandList())) { + exps.add( + Pair.of( + tmpBb.convertExpression(operand.e), + validator.deriveAlias(operand.e, operand.i))); + } + RelNode in = + (null == tmpBb.root) + ? LogicalValues.createOneRow(cluster) + : tmpBb.root; + unionRels.add( + RelOptUtil.createProject( + in, + Pair.left(exps), + Pair.right(exps), + true)); } - /** - * Converts an EXISTS or IN predicate into a join. For EXISTS, the subquery - * produces an indicator variable, and the result is a relational expression - * which outer joins that indicator to the original query. After performing - * the outer join, the condition will be TRUE if the EXISTS condition holds, - * NULL otherwise. - * - * @param seek A query, for example 'select * from emp' or - * 'values (1,2,3)' or '('Foo', 34)'. - * @param subqueryType Whether sub-query is IN, EXISTS or scalar - * @param logic Whether the answer needs to be in full 3-valued logic (TRUE, - * FALSE, UNKNOWN) will be required, or whether we can accept an - * approximation (say representing UNKNOWN as FALSE) - * @param needsOuterJoin Whether an outer join is needed - * @return join expression - * @pre extraExpr == null || extraName != null - */ - private Pair convertExists(SqlNode seek, RelOptUtil.SubqueryType subqueryType, RelOptUtil.Logic logic, boolean needsOuterJoin) { - final SqlValidatorScope seekScope = (seek instanceof SqlSelect) ? validator.getSelectScope((SqlSelect) seek) : null; - final Blackboard seekBb = createBlackboard(seekScope, null); - RelNode seekRel = convertQueryOrInList(seekBb, seek); - - return RelOptUtil.createExistsPlan(seekRel, subqueryType, logic, needsOuterJoin); - } - - private RelNode convertQueryOrInList(Blackboard bb, SqlNode seek) { - // NOTE: Once we start accepting single-row queries as row constructors, - // there will be an ambiguity here for a case like X IN ((SELECT Y FROM - // Z)). The SQL standard resolves the ambiguity by saying that a lone - // select should be interpreted as a table expression, not a row - // expression. The semantic difference is that a table expression can - // return multiple rows. - if (seek instanceof SqlNodeList) { - return convertRowValues(bb, seek, ((SqlNodeList) seek).getList(), false, null); - } else { - return convertQueryRecursive(seek, false, null); - } + if (unionRels.size() == 0) { + throw Util.newInternal("empty values clause"); + } else if (unionRels.size() == 1) { + bb.setRoot( + unionRels.get(0), + true); + } else { + bb.setRoot( + LogicalUnion.create(unionRels, true), + true); } - private RelNode convertRowValues(Blackboard bb, SqlNode rowList, Collection rows, boolean allowLiteralsOnly, RelDataType targetRowType) { - // NOTE jvs 30-Apr-2006: We combine all rows consisting entirely of - // literals into a single LogicalValues; this gives the optimizer a smaller - // input tree. For everything else (computed expressions, row - // subqueries), we union each row in as a projection on top of a - // LogicalOneRow. + // REVIEW jvs 22-Jan-2004: should I add + // mapScopeToLux.put(validator.getScope(values),bb.root); + // ? + } - final ImmutableList.Builder> tupleList = ImmutableList.builder(); - final RelDataType rowType; - if (targetRowType != null) { - rowType = targetRowType; - } else { - rowType = SqlTypeUtil.promoteToRowType(typeFactory, validator.getValidatedNodeType(rowList), null); - } + //~ Inner Classes ---------------------------------------------------------- - final List unionInputs = new ArrayList<>(); - for (SqlNode node : rows) { - SqlBasicCall call; - if (isRowConstructor(node)) { - call = (SqlBasicCall) node; - ImmutableList.Builder tuple = ImmutableList.builder(); - for (Ord operand : Ord.zip(call.operands)) { - RexLiteral rexLiteral = convertLiteralInValuesList(operand.e, bb, rowType, operand.i); - if ((rexLiteral == null) && allowLiteralsOnly) { - return null; - } - if ((rexLiteral == null) || !shouldCreateValuesRel) { - // fallback to convertRowConstructor - tuple = null; - break; - } - tuple.add(rexLiteral); - } - if (tuple != null) { - tupleList.add(tuple.build()); - continue; - } - } else { - RexLiteral rexLiteral = convertLiteralInValuesList(node, bb, rowType, 0); - if ((rexLiteral != null) && shouldCreateValuesRel) { - tupleList.add(ImmutableList.of(rexLiteral)); - continue; - } else { - if ((rexLiteral == null) && allowLiteralsOnly) { - return null; - } - } + /** + * Workspace for translating an individual SELECT statement (or sub-SELECT). + */ + protected class Blackboard implements SqlRexContext, SqlVisitor { + /** + * Collection of {@link RelNode} objects which correspond to a SELECT + * statement. + */ + public final SqlValidatorScope scope; + private final Map nameToNodeMap; + public RelNode root; + private List inputs; + private final Map mapCorrelateToRex = + new HashMap<>(); - // convert "1" to "row(1)" - call = (SqlBasicCall) SqlStdOperatorTable.ROW.createCall(SqlParserPos.ZERO, node); - } - unionInputs.add(convertRowConstructor(bb, call)); - } - LogicalValues values = LogicalValues.create(cluster, rowType, tupleList.build()); - RelNode resultRel; - if (unionInputs.isEmpty()) { - resultRel = values; - } else { - if (!values.getTuples().isEmpty()) { - unionInputs.add(values); - } - resultRel = LogicalUnion.create(unionInputs, true); - } - leaves.add(resultRel); - return resultRel; - } + final List cursors = new ArrayList<>(); - private RexLiteral convertLiteralInValuesList(SqlNode sqlNode, Blackboard bb, RelDataType rowType, int iField) { - if (!(sqlNode instanceof SqlLiteral)) { - return null; - } - RelDataTypeField field = rowType.getFieldList().get(iField); - RelDataType type = field.getType(); - if (type.isStruct()) { - // null literals for weird stuff like UDT's need - // special handling during type flattening, so - // don't use LogicalValues for those - return null; - } + /** + * List of IN and EXISTS nodes inside this + * SELECT statement (but not inside sub-queries). + */ + private final Set subqueryList = Sets.newLinkedHashSet(); - RexNode literalExpr = exprConverter.convertLiteral(bb, (SqlLiteral) sqlNode); + private boolean subqueryNeedsOuterJoin; - if (!(literalExpr instanceof RexLiteral)) { - assert literalExpr.isA(SqlKind.CAST); - RexNode child = ((RexCall) literalExpr).getOperands().get(0); - assert RexLiteral.isNullLiteral(child); + /** + * Workspace for building aggregates. + */ + AggConverter agg; - // NOTE jvs 22-Nov-2006: we preserve type info - // in LogicalValues digest, so it's OK to lose it here - return (RexLiteral) child; - } + /** + * When converting window aggregate, we need to know if the window is + * guaranteed to be non-empty. + */ + SqlWindow window; - RexLiteral literal = (RexLiteral) literalExpr; + /** + * Project the groupby expressions out of the root of this sub-select. + * Subqueries can reference group by expressions projected from the + * "right" to the subquery. + */ + private final Map> + mapRootRelToFieldProjection = new HashMap<>(); - Comparable value = literal.getValue(); + private final List columnMonotonicities = + new ArrayList<>(); - if (SqlTypeUtil.isExactNumeric(type)) { - BigDecimal roundedValue = NumberUtil.rescaleBigDecimal((BigDecimal) value, type.getScale()); - return rexBuilder.makeExactLiteral(roundedValue, type); - } + private final List systemFieldList = new ArrayList<>(); + final boolean top; - if ((value instanceof NlsString) && (type.getSqlTypeName() == SqlTypeName.CHAR)) { - // pad fixed character type - NlsString unpadded = (NlsString) value; - return rexBuilder.makeCharLiteral(new NlsString(Util.rpad(unpadded.getValue(), type.getPrecision()), unpadded.getCharsetName(), unpadded.getCollation())); - } - return literal; + /** + * Creates a Blackboard. + * + * @param scope Name-resolution scope for expressions validated + * within this query. Can be null if this Blackboard is + * for a leaf node, say + * @param nameToNodeMap Map which translates the expression to map a + * given parameter into, if translating expressions; + * null otherwise + * @param top Whether this is the root of the query + */ + protected Blackboard(SqlValidatorScope scope, + Map nameToNodeMap, boolean top) { + this.scope = scope; + this.nameToNodeMap = nameToNodeMap; + this.top = top; + subqueryNeedsOuterJoin = false; } - private boolean isRowConstructor(SqlNode node) { - if (!(node.getKind() == SqlKind.ROW)) { - return false; - } - SqlCall call = (SqlCall) node; - return call.getOperator().getName().equalsIgnoreCase("row"); + public RexNode register( + RelNode rel, + JoinRelType joinType) { + return register(rel, joinType, null); } /** - * Builds a list of all IN or EXISTS operators - * inside SQL parse tree. Does not traverse inside queries. + * Registers a relational expression. * - * @param bb blackboard - * @param node the SQL parse tree - * @param logic Whether the answer needs to be in full 3-valued logic (TRUE, - * FALSE, UNKNOWN) will be required, or whether we can accept - * an approximation (say representing UNKNOWN as FALSE) - * @param registerOnlyScalarSubqueries if set to true and the parse tree - * corresponds to a variation of a select - * node, only register it if it's a scalar - * subquery + * @param rel Relational expression + * @param joinType Join type + * @param leftKeys LHS of IN clause, or null for expressions + * other than IN + * @return Expression with which to refer to the row (or partial row) + * coming from this relational expression's side of the join */ - private void findSubqueries(Blackboard bb, SqlNode node, RelOptUtil.Logic logic, boolean registerOnlyScalarSubqueries) { - final SqlKind kind = node.getKind(); - switch (kind) { - case EXISTS: - case SELECT: - case MULTISET_QUERY_CONSTRUCTOR: - case MULTISET_VALUE_CONSTRUCTOR: - case CURSOR: - case SCALAR_QUERY: - if (!registerOnlyScalarSubqueries || (kind == SqlKind.SCALAR_QUERY)) { - bb.registerSubquery(node, RelOptUtil.Logic.TRUE_FALSE); - } - return; - case IN: - if (((SqlCall) node).getOperator() == SqlStdOperatorTable.NOT_IN) { - logic = logic.negate(); - } - break; - case NOT: - logic = logic.negate(); - break; - } - if (node instanceof SqlCall) { - if (kind == SqlKind.OR || kind == SqlKind.NOT) { - // It's always correct to outer join subquery with - // containing query; however, when predicates involve Or - // or NOT, outer join might be necessary. - bb.subqueryNeedsOuterJoin = true; - } - for (SqlNode operand : ((SqlCall) node).getOperandList()) { - if (operand != null) { - // In the case of an IN expression, locate scalar - // subqueries so we can convert them to constants - findSubqueries(bb, operand, logic, kind == SqlKind.IN || registerOnlyScalarSubqueries); - } - } - } else if (node instanceof SqlNodeList) { - for (SqlNode child : (SqlNodeList) node) { - findSubqueries(bb, child, logic, kind == SqlKind.IN || registerOnlyScalarSubqueries); - } - } + public RexNode register( + RelNode rel, + JoinRelType joinType, + List leftKeys) { + assert joinType != null; + if (root == null) { + assert leftKeys == null; + setRoot(rel, false); + return rexBuilder.makeRangeReference( + root.getRowType(), + 0, + false); + } + + final RexNode joinCond; + final int origLeftInputCount = root.getRowType().getFieldCount(); + if (leftKeys != null) { + List newLeftInputExpr = Lists.newArrayList(); + for (int i = 0; i < origLeftInputCount; i++) { + newLeftInputExpr.add(rexBuilder.makeInputRef(root, i)); + } + + final List leftJoinKeys = Lists.newArrayList(); + for (RexNode leftKey : leftKeys) { + int index = newLeftInputExpr.indexOf(leftKey); + if (index < 0 || joinType == JoinRelType.LEFT) { + index = newLeftInputExpr.size(); + newLeftInputExpr.add(leftKey); + } + leftJoinKeys.add(index); + } + + RelNode newLeftInput = + RelOptUtil.createProject( + root, + newLeftInputExpr, + null, + true); + + // maintain the group by mapping in the new LogicalProject + if (mapRootRelToFieldProjection.containsKey(root)) { + mapRootRelToFieldProjection.put( + newLeftInput, + mapRootRelToFieldProjection.get(root)); + } + + setRoot(newLeftInput, false); + + // right fields appear after the LHS fields. + final int rightOffset = root.getRowType().getFieldCount() + - newLeftInput.getRowType().getFieldCount(); + final List rightKeys = + Util.range(rightOffset, rightOffset + leftKeys.size()); + + joinCond = + RelOptUtil.createEquiJoinCondition(newLeftInput, leftJoinKeys, + rel, rightKeys, rexBuilder); + } else { + joinCond = rexBuilder.makeLiteral(true); + } + + int leftFieldCount = root.getRowType().getFieldCount(); + final RelNode join = + createJoin( + this, + root, + rel, + joinCond, + joinType); + + setRoot(join, false); + + if (leftKeys != null + && joinType == JoinRelType.LEFT) { + final int leftKeyCount = leftKeys.size(); + int rightFieldLength = rel.getRowType().getFieldCount(); + assert leftKeyCount == rightFieldLength - 1; + + final int rexRangeRefLength = leftKeyCount + rightFieldLength; + RelDataType returnType = + typeFactory.createStructType( + new AbstractList>() { + public Map.Entry get( + int index) { + return join.getRowType().getFieldList() + .get(origLeftInputCount + index); + } + + public int size() { + return rexRangeRefLength; + } + }); - // Now that we've located any scalar subqueries inside the IN - // expression, register the IN expression itself. We need to - // register the scalar subqueries first so they can be converted - // before the IN expression is converted. - if (kind == SqlKind.IN) { - if (logic == RelOptUtil.Logic.TRUE_FALSE_UNKNOWN && !validator.getValidatedNodeType(node).isNullable()) { - logic = RelOptUtil.Logic.UNKNOWN_AS_FALSE; - } - // TODO: This conversion is only valid in the WHERE clause - if (logic == RelOptUtil.Logic.UNKNOWN_AS_FALSE && !bb.subqueryNeedsOuterJoin) { - logic = RelOptUtil.Logic.TRUE; - } - bb.registerSubquery(node, logic); - } + return rexBuilder.makeRangeReference( + returnType, + origLeftInputCount, + false); + } else { + return rexBuilder.makeRangeReference( + rel.getRowType(), + leftFieldCount, + joinType.generatesNullsOnRight()); + } } /** - * Converts an expression from {@link SqlNode} to {@link RexNode} format. + * Sets a new root relational expression, as the translation process + * backs its way further up the tree. * - * @param node Expression to translate - * @return Converted expression + * @param root New root relational expression + * @param leaf Whether the relational expression is a leaf, that is, + * derived from an atomic relational expression such as a table + * name in the from clause, or the projection on top of a + * select-subquery. In particular, relational expressions + * derived from JOIN operators are not leaves, but set + * expressions are. */ - public RexNode convertExpression(SqlNode node) { - Map nameToTypeMap = Collections.emptyMap(); - Blackboard bb = createBlackboard(new ParameterScope((SqlValidatorImpl) validator, nameToTypeMap), null); - return bb.convertExpression(node); + public void setRoot(RelNode root, boolean leaf) { + setRoot( + Collections.singletonList(root), root, root instanceof LogicalJoin); + if (leaf) { + leaves.add(root); + } + this.columnMonotonicities.clear(); + } + + private void setRoot( + List inputs, + RelNode root, + boolean hasSystemFields) { + this.inputs = inputs; + this.root = root; + this.systemFieldList.clear(); + if (hasSystemFields) { + this.systemFieldList.addAll(getSystemFields()); + } } /** - * Converts an expression from {@link SqlNode} to {@link RexNode} format, - * mapping identifier references to predefined expressions. + * Notifies this Blackboard that the root just set using + * {@link #setRoot(RelNode, boolean)} was derived using dataset + * substitution. * - * @param node Expression to translate - * @param nameToNodeMap map from String to {@link RexNode}; when an - * {@link SqlIdentifier} is encountered, it is used as a - * key and translated to the corresponding value from - * this map - * @return Converted expression + *

The default implementation is not interested in such + * notifications, and does nothing. + * + * @param datasetName Dataset name */ - public RexNode convertExpression(SqlNode node, Map nameToNodeMap) { - final Map nameToTypeMap = new HashMap<>(); - for (Map.Entry entry : nameToNodeMap.entrySet()) { - nameToTypeMap.put(entry.getKey(), entry.getValue().getType()); - } - Blackboard bb = createBlackboard(new ParameterScope((SqlValidatorImpl) validator, nameToTypeMap), nameToNodeMap); - return bb.convertExpression(node); + public void setDataset(String datasetName) { + } + + void setRoot(List inputs) { + setRoot(inputs, null, false); } /** - * Converts a non-standard expression. + * Returns an expression with which to reference a from-list item. * - *

This method is an extension-point that derived classes can override. If - * this method returns a null result, the normal expression translation - * process will proceed. The default implementation always returns null. - * - * @param node Expression - * @param bb Blackboard - * @return null to proceed with the usual expression translation process + * @param qualified the alias of the from item + * @return a {@link RexFieldAccess} or {@link RexRangeRef}, or null if + * not found */ - protected RexNode convertExtendedExpression(SqlNode node, Blackboard bb) { + RexNode lookupExp(SqlQualified qualified) { + if (nameToNodeMap != null && qualified.prefixLength == 1) { + RexNode node = nameToNodeMap.get(qualified.identifier.names.get(0)); + if (node == null) { + throw Util.newInternal("Unknown identifier '" + qualified.identifier + + "' encountered while expanding expression"); + } + return node; + } + int[] offsets = {-1}; + final SqlValidatorScope[] ancestorScopes = {null}; + SqlValidatorNamespace foundNs = + scope.resolve(qualified.prefix(), ancestorScopes, offsets); + if (foundNs == null) { return null; + } + + // Found in current query's from list. Find which from item. + // We assume that the order of the from clause items has been + // preserved. + SqlValidatorScope ancestorScope = ancestorScopes[0]; + boolean isParent = ancestorScope != scope; + if ((inputs != null) && !isParent) { + int offset = offsets[0]; + final LookupContext rels = + new LookupContext(this, inputs, systemFieldList.size()); + return lookup(offset, rels); + } else { + // We're referencing a relational expression which has not been + // converted yet. This occurs when from items are correlated, + // e.g. "select from emp as emp join emp.getDepts() as dept". + // Create a temporary expression. + assert isParent; + DeferredLookup lookup = + new DeferredLookup(this, qualified.identifier.names.get(0)); + final CorrelationId correlName = cluster.createCorrel(); + mapCorrelToDeferred.put(correlName, lookup); + final RelDataType rowType = foundNs.getRowType(); + return rexBuilder.makeCorrel(rowType, correlName); + } } - private RexNode convertOver(Blackboard bb, SqlNode node) { - SqlCall call = (SqlCall) node; - SqlCall aggCall = call.operand(0); - SqlNode windowOrRef = call.operand(1); - final SqlWindow window = validator.resolveWindow(windowOrRef, bb.scope, true); - // ROW_NUMBER() expects specific kind of framing. - if (aggCall.getOperator() == SqlStdOperatorTable.ROW_NUMBER) { - window.setLowerBound(SqlWindow.createUnboundedPreceding(SqlParserPos.ZERO)); - window.setUpperBound(SqlWindow.createCurrentRow(SqlParserPos.ZERO)); - window.setRows(SqlLiteral.createBoolean(true, SqlParserPos.ZERO)); - } - final SqlNodeList partitionList = window.getPartitionList(); - final ImmutableList.Builder partitionKeys = ImmutableList.builder(); - for (SqlNode partition : partitionList) { - partitionKeys.add(bb.convertExpression(partition)); - } - RexNode lowerBound = bb.convertExpression(window.getLowerBound()); - RexNode upperBound = bb.convertExpression(window.getUpperBound()); - SqlNodeList orderList = window.getOrderList(); - if ((orderList.size() == 0) && !window.isRows()) { - // A logical range requires an ORDER BY clause. Use the implicit - // ordering of this relation. There must be one, otherwise it would - // have failed validation. - orderList = bb.scope.getOrderList(); - if (orderList == null) { - throw new AssertionError("Relation should have sort key for implicit ORDER BY"); - } - } - final ImmutableList.Builder orderKeys = ImmutableList.builder(); - final Set flags = EnumSet.noneOf(SqlKind.class); - for (SqlNode order : orderList) { - flags.clear(); - RexNode e = bb.convertSortExpression(order, flags); - orderKeys.add(new RexFieldCollation(e, flags)); + /** + * Creates an expression with which to reference the expression whose + * offset in its from-list is {@code offset}. + */ + RexNode lookup( + int offset, + LookupContext lookupContext) { + Pair pair = lookupContext.findRel(offset); + return rexBuilder.makeRangeReference( + pair.left.getRowType(), + pair.right, + false); + } + + RelDataTypeField getRootField(RexInputRef inputRef) { + int fieldOffset = inputRef.getIndex(); + for (RelNode input : inputs) { + RelDataType rowType = input.getRowType(); + if (rowType == null) { + // TODO: remove this once leastRestrictive + // is correctly implemented + return null; + } + if (fieldOffset < rowType.getFieldCount()) { + return rowType.getFieldList().get(fieldOffset); + } + fieldOffset -= rowType.getFieldCount(); + } + throw new AssertionError(); + } + + public void flatten( + List rels, + int systemFieldCount, + int[] start, + List> relOffsetList) { + for (RelNode rel : rels) { + if (leaves.contains(rel)) { + relOffsetList.add( + Pair.of(rel, start[0])); + start[0] += rel.getRowType().getFieldCount(); + } else { + if (rel instanceof LogicalJoin + || rel instanceof LogicalAggregate) { + start[0] += systemFieldCount; + } + flatten( + rel.getInputs(), + systemFieldCount, + start, + relOffsetList); + } + } + } + + void registerSubquery(SqlNode node, RelOptUtil.Logic logic) { + for (SubQuery subQuery : subqueryList) { + if (node.equalsDeep(subQuery.node, false)) { + return; } - try { - Util.permAssert(bb.window == null, "already in window agg mode"); - bb.window = window; - RexNode rexAgg = exprConverter.convertCall(bb, aggCall); - rexAgg = rexBuilder.ensureType(validator.getValidatedNodeType(call), rexAgg, false); - - // Walk over the tree and apply 'over' to all agg functions. This is - // necessary because the returned expression is not necessarily a call - // to an agg function. For example, AVG(x) becomes SUM(x) / COUNT(x). - final RexShuttle visitor = new HistogramShuttle(partitionKeys.build(), orderKeys.build(), RexWindowBound.create(window.getLowerBound(), lowerBound), RexWindowBound.create(window.getUpperBound(), upperBound), window); - return rexAgg.accept(visitor); - } finally { - bb.window = null; + } + subqueryList.add(new SubQuery(node, logic)); + } + + SubQuery getSubquery(SqlNode expr) { + for (SubQuery subQuery : subqueryList) { + if (expr.equalsDeep(subQuery.node, false)) { + return subQuery; } + } + + return null; } - /** - * Converts a FROM clause into a relational expression. - * - * @param bb Scope within which to resolve identifiers - * @param from FROM clause of a query. Examples include: - * - *

    - *
  • a single table ("SALES.EMP"), - *
  • an aliased table ("EMP AS E"), - *
  • a list of tables ("EMP, DEPT"), - *
  • an ANSI Join expression ("EMP JOIN DEPT ON EMP.DEPTNO = - * DEPT.DEPTNO"), - *
  • a VALUES clause ("VALUES ('Fred', 20)"), - *
  • a query ("(SELECT * FROM EMP WHERE GENDER = 'F')"), - *
  • or any combination of the above. - *
- */ - protected void convertFrom(Blackboard bb, SqlNode from) { - SqlCall call; - final SqlNode[] operands; - switch (from.getKind()) { - case AS: - operands = ((SqlBasicCall) from).getOperands(); - convertFrom(bb, operands[0]); - return; - - case WITH_ITEM: - convertFrom(bb, ((SqlWithItem) from).query); - return; - - case WITH: - convertFrom(bb, ((SqlWith) from).body); - return; - - case TABLESAMPLE: - operands = ((SqlBasicCall) from).getOperands(); - SqlSampleSpec sampleSpec = SqlLiteral.sampleValue(operands[1]); - if (sampleSpec instanceof SqlSampleSpec.SqlSubstitutionSampleSpec) { - String sampleName = ((SqlSampleSpec.SqlSubstitutionSampleSpec) sampleSpec).getName(); - datasetStack.push(sampleName); - convertFrom(bb, operands[0]); - datasetStack.pop(); - } else if (sampleSpec instanceof SqlSampleSpec.SqlTableSampleSpec) { - SqlSampleSpec.SqlTableSampleSpec tableSampleSpec = (SqlSampleSpec.SqlTableSampleSpec) sampleSpec; - convertFrom(bb, operands[0]); - RelOptSamplingParameters params = new RelOptSamplingParameters(tableSampleSpec.isBernoulli(), tableSampleSpec.getSamplePercentage(), tableSampleSpec.isRepeatable(), tableSampleSpec.getRepeatableSeed()); - bb.setRoot(new Sample(cluster, bb.root, params), false); - } else { - throw Util.newInternal("unknown TABLESAMPLE type: " + sampleSpec); - } - return; + ImmutableList retrieveCursors() { + try { + return ImmutableList.copyOf(cursors); + } finally { + cursors.clear(); + } + } - case IDENTIFIER: - final SqlValidatorNamespace fromNamespace = validator.getNamespace(from).resolve(); - if (fromNamespace.getNode() != null) { - convertFrom(bb, fromNamespace.getNode()); - return; - } - final String datasetName = datasetStack.isEmpty() ? null : datasetStack.peek(); - boolean[] usedDataset = { false }; - RelOptTable table = SqlValidatorUtil.getRelOptTable(fromNamespace, catalogReader, datasetName, usedDataset); - final RelNode tableRel; - if (shouldConvertTableAccess) { - tableRel = toRel(table); - } else { - tableRel = LogicalTableScan.create(cluster, table); - } - bb.setRoot(tableRel, true); - if (usedDataset[0]) { - bb.setDataset(datasetName); + public RexNode convertExpression(SqlNode expr) { + // If we're in aggregation mode and this is an expression in the + // GROUP BY clause, return a reference to the field. + if (agg != null) { + final SqlNode expandedGroupExpr = validator.expand(expr, scope); + final int ref = agg.lookupGroupExpr(expandedGroupExpr); + if (ref >= 0) { + return rexBuilder.makeInputRef(root, ref); + } + if (expr instanceof SqlCall) { + final RexNode rex = agg.lookupAggregates((SqlCall) expr); + if (rex != null) { + return rex; + } + } + } + + // Allow the derived class chance to override the standard + // behavior for special kinds of expressions. + RexNode rex = convertExtendedExpression(expr, this); + if (rex != null) { + return rex; + } + + // Sub-queries and OVER expressions are not like ordinary + // expressions. + final SqlKind kind = expr.getKind(); + final SubQuery subQuery; + if (!expand) { + final SqlCall call; + final SqlNode query; + final RelRoot root; + switch (kind) { + case IN: + call = (SqlCall) expr; + query = call.operand(1); + if (!(query instanceof SqlNodeList)) { + final SqlInOperator op = (SqlInOperator) call.getOperator(); + root = convertQueryRecursive(query, false, null); + final SqlNode operand = call.operand(0); + List nodes; + switch (operand.getKind()) { + case ROW: + nodes = ((SqlCall) operand).getOperandList(); + break; + default: + nodes = ImmutableList.of(operand); } - return; - - case JOIN: - final SqlJoin join = (SqlJoin) from; - final Blackboard fromBlackboard = createBlackboard(validator.getJoinScope(from), null); - SqlNode left = join.getLeft(); - SqlNode right = join.getRight(); - final boolean isNatural = join.isNatural(); - final JoinType joinType = join.getJoinType(); - final Blackboard leftBlackboard = createBlackboard(Util.first(validator.getJoinScope(left), ((DelegatingScope) bb.scope).getParent()), null); - final Blackboard rightBlackboard = createBlackboard(Util.first(validator.getJoinScope(right), ((DelegatingScope) bb.scope).getParent()), null); - convertFrom(leftBlackboard, left); - RelNode leftRel = leftBlackboard.root; - convertFrom(rightBlackboard, right); - RelNode rightRel = rightBlackboard.root; - JoinRelType convertedJoinType = convertJoinType(joinType); - RexNode conditionExp; - final SqlValidatorNamespace leftNamespace = validator.getNamespace(left); - final SqlValidatorNamespace rightNamespace = validator.getNamespace(right); - if (isNatural) { - final RelDataType leftRowType = leftNamespace.getRowType(); - final RelDataType rightRowType = rightNamespace.getRowType(); - final List columnList = SqlValidatorUtil.deriveNaturalJoinColumnList(leftRowType, rightRowType); - conditionExp = convertUsing(leftNamespace, rightNamespace, columnList); - } else { - conditionExp = convertJoinCondition(fromBlackboard, leftNamespace, rightNamespace, join.getCondition(), join.getConditionType(), leftRel, rightRel); + final ImmutableList.Builder builder = + ImmutableList.builder(); + for (SqlNode node : nodes) { + builder.add(convertExpression(node)); } + final RexSubQuery in = RexSubQuery.in(root.rel, builder.build()); + return op.isNotIn() + ? rexBuilder.makeCall(SqlStdOperatorTable.NOT, in) + : in; + } + break; - final RelNode joinRel = createJoin(fromBlackboard, leftRel, rightRel, conditionExp, convertedJoinType); - bb.setRoot(joinRel, false); - return; - - case SELECT: - case INTERSECT: - case EXCEPT: - case UNION: - final RelNode rel = convertQueryRecursive(from, false, null); - bb.setRoot(rel, true); - return; - - case VALUES: - convertValuesImpl(bb, (SqlCall) from, null); - return; - - case UNNEST: - final SqlNode node = ((SqlCall) from).operand(0); - replaceSubqueries(bb, node, RelOptUtil.Logic.TRUE_FALSE_UNKNOWN); - final RelNode childRel = RelOptUtil.createProject((null != bb.root) ? bb.root : LogicalValues.createOneRow(cluster), Collections.singletonList(bb.convertExpression(node)), Collections.singletonList(validator.deriveAlias(node, 0)), true); - - Uncollect uncollect = new Uncollect(cluster, cluster.traitSetOf(Convention.NONE), childRel); - bb.setRoot(uncollect, true); - return; - - case COLLECTION_TABLE: - call = (SqlCall) from; - - // Dig out real call; TABLE() wrapper is just syntactic. - assert call.getOperandList().size() == 1; - call = call.operand(0); - convertCollectionTable(bb, call); - return; - - default: - throw Util.newInternal("not a join operator " + from); - } + case EXISTS: + call = (SqlCall) expr; + query = Iterables.getOnlyElement(call.getOperandList()); + root = convertQueryRecursive(query, false, null); + RelNode rel = root.rel; + while (rel instanceof Project + || rel instanceof Sort + && ((Sort) rel).fetch == null + && ((Sort) rel).offset == null) { + rel = ((SingleRel) rel).getInput(); + } + return RexSubQuery.exists(rel); + + case SCALAR_QUERY: + call = (SqlCall) expr; + query = Iterables.getOnlyElement(call.getOperandList()); + root = convertQueryRecursive(query, false, null); + return RexSubQuery.scalar(root.rel); + } + } + + switch (kind) { + case CURSOR: + case IN: + subQuery = getSubquery(expr); + + assert subQuery != null; + rex = subQuery.expr; + assert rex != null : "rex != null"; + return rex; + + case SELECT: + case EXISTS: + case SCALAR_QUERY: + subQuery = getSubquery(expr); + assert subQuery != null; + rex = subQuery.expr; + assert rex != null : "rex != null"; + + if (((kind == SqlKind.SCALAR_QUERY) + || (kind == SqlKind.EXISTS)) + && isConvertedSubq(rex)) { + // scalar subquery or EXISTS has been converted to a + // constant + return rex; + } + + // The indicator column is the last field of the subquery. + RexNode fieldAccess = + rexBuilder.makeFieldAccess( + rex, + rex.getType().getFieldCount() - 1); + + // The indicator column will be nullable if it comes from + // the null-generating side of the join. For EXISTS, add an + // "IS TRUE" check so that the result is "BOOLEAN NOT NULL". + if (fieldAccess.getType().isNullable() + && kind == SqlKind.EXISTS) { + fieldAccess = + rexBuilder.makeCall( + SqlStdOperatorTable.IS_NOT_NULL, + fieldAccess); + } + return fieldAccess; + + case OVER: + return convertOver(this, expr); + + default: + // fall through + } + + // Apply standard conversions. + rex = expr.accept(this); + Util.permAssert(rex != null, "conversion result not null"); + return rex; } - protected void convertCollectionTable(Blackboard bb, SqlCall call) { - final SqlOperator operator = call.getOperator(); - if (operator == SqlStdOperatorTable.TABLESAMPLE) { - final String sampleName = SqlLiteral.stringValue(call.operand(0)); - datasetStack.push(sampleName); - SqlCall cursorCall = call.operand(1); - SqlNode query = cursorCall.operand(0); - RelNode converted = convertQuery(query, false, false); - bb.setRoot(converted, false); - datasetStack.pop(); - return; - } - replaceSubqueries(bb, call, RelOptUtil.Logic.TRUE_FALSE_UNKNOWN); - - // Expand table macro if possible. It's more efficient than - // LogicalTableFunctionScan. - if (operator instanceof SqlUserDefinedTableMacro) { - final SqlUserDefinedTableMacro udf = (SqlUserDefinedTableMacro) operator; - final TranslatableTable table = udf.getTable(typeFactory, call.getOperandList()); - final RelDataType rowType = table.getRowType(typeFactory); - RelOptTable relOptTable = RelOptTableImpl.create(null, rowType, table); - RelNode converted = toRel(relOptTable); - bb.setRoot(converted, true); - return; - } + /** + * Converts an item in an ORDER BY clause, extracting DESC, NULLS LAST + * and NULLS FIRST flags first. + */ + public RexNode convertSortExpression(SqlNode expr, Set flags) { + switch (expr.getKind()) { + case DESCENDING: + case NULLS_LAST: + case NULLS_FIRST: + flags.add(expr.getKind()); + final SqlNode operand = ((SqlCall) expr).operand(0); + return convertSortExpression(operand, flags); + default: + return convertExpression(expr); + } + } - Type elementType; - if (operator instanceof SqlUserDefinedTableFunction) { - SqlUserDefinedTableFunction udtf = (SqlUserDefinedTableFunction) operator; - elementType = udtf.getElementType(typeFactory, call.getOperandList()); - } else { - elementType = null; + /** + * Determines whether a RexNode corresponds to a subquery that's been + * converted to a constant. + * + * @param rex the expression to be examined + * @return true if the expression is a dynamic parameter, a literal, or + * a literal that is being cast + */ + private boolean isConvertedSubq(RexNode rex) { + if ((rex instanceof RexLiteral) + || (rex instanceof RexDynamicParam)) { + return true; + } + if (rex instanceof RexCall) { + RexCall call = (RexCall) rex; + if (call.getOperator() == SqlStdOperatorTable.CAST) { + RexNode operand = call.getOperands().get(0); + if (operand instanceof RexLiteral) { + return true; + } } + } + return false; + } - RexNode rexCall = bb.convertExpression(call); - final List inputs = bb.retrieveCursors(); - Set columnMappings = getColumnMappings(operator); - LogicalTableFunctionScan callRel = LogicalTableFunctionScan.create(cluster, inputs, rexCall, elementType, validator.getValidatedNodeType(call), columnMappings); - bb.setRoot(callRel, true); - afterTableFunction(bb, call, callRel); + // implement SqlRexContext + public int getGroupCount() { + if (agg != null) { + return agg.groupExprs.size(); + } + if (window != null) { + return window.isAlwaysNonEmpty() ? 1 : 0; + } + return -1; } - protected void afterTableFunction(SqlToRelConverter.Blackboard bb, SqlCall call, LogicalTableFunctionScan callRel) { + // implement SqlRexContext + public RexBuilder getRexBuilder() { + return rexBuilder; } - private Set getColumnMappings(SqlOperator op) { - SqlReturnTypeInference rti = op.getReturnTypeInference(); - if (rti == null) { - return null; - } - if (rti instanceof TableFunctionReturnTypeInference) { - TableFunctionReturnTypeInference tfrti = (TableFunctionReturnTypeInference) rti; - return tfrti.getColumnMappings(); - } else { - return null; - } + // implement SqlRexContext + public RexRangeRef getSubqueryExpr(SqlCall call) { + final SubQuery subQuery = getSubquery(call); + assert subQuery != null; + return (RexRangeRef) subQuery.expr; } - protected RelNode createJoin(Blackboard bb, RelNode leftRel, RelNode rightRel, RexNode joinCond, JoinRelType joinType) { - assert joinCond != null; + // implement SqlRexContext + public RelDataTypeFactory getTypeFactory() { + return typeFactory; + } - Set correlatedVariables = RelOptUtil.getVariablesUsed(rightRel); - if (correlatedVariables.size() > 0) { - final ImmutableBitSet.Builder requiredColumns = ImmutableBitSet.builder(); - final List correlNames = Lists.newArrayList(); + // implement SqlRexContext + public DefaultValueFactory getDefaultValueFactory() { + return defaultValueFactory; + } - // All correlations must refer the same namespace since correlation - // produces exactly one correlation source. - // The same source might be referenced by different variables since - // DeferredLookups are not de-duplicated at create time. - SqlValidatorNamespace prevNs = null; + // implement SqlRexContext + public SqlValidator getValidator() { + return validator; + } - for (String correlName : correlatedVariables) { - DeferredLookup lookup = mapCorrelToDeferred.get(correlName); - RexFieldAccess fieldAccess = lookup.getFieldAccess(correlName); - String originalRelName = lookup.getOriginalRelName(); - String originalFieldName = fieldAccess.getField().getName(); + // implement SqlRexContext + public RexNode convertLiteral(SqlLiteral literal) { + return exprConverter.convertLiteral(this, literal); + } - int[] nsIndexes = { -1 }; - final SqlValidatorScope[] ancestorScopes = { null }; - SqlValidatorNamespace foundNs = lookup.bb.scope.resolve(ImmutableList.of(originalRelName), ancestorScopes, nsIndexes); + public RexNode convertInterval(SqlIntervalQualifier intervalQualifier) { + return exprConverter.convertInterval(this, intervalQualifier); + } - assert foundNs != null; - assert nsIndexes.length == 1; + // implement SqlVisitor + public RexNode visit(SqlLiteral literal) { + return exprConverter.convertLiteral(this, literal); + } - int childNamespaceIndex = nsIndexes[0]; + // implement SqlVisitor + public RexNode visit(SqlCall call) { + if (agg != null) { + final SqlOperator op = call.getOperator(); + if (window == null + && (op.isAggregator() || op.getKind() == SqlKind.FILTER)) { + return agg.lookupAggregates(call); + } + } + return exprConverter.convertCall(this, + new SqlCallBinding(validator, scope, call).permutedCall()); + } - SqlValidatorScope ancestorScope = ancestorScopes[0]; - boolean correlInCurrentScope = ancestorScope == bb.scope; + // implement SqlVisitor + public RexNode visit(SqlNodeList nodeList) { + throw new UnsupportedOperationException(); + } - if (!correlInCurrentScope) { - continue; - } + // implement SqlVisitor + public RexNode visit(SqlIdentifier id) { + return convertIdentifier(this, id); + } - if (prevNs == null) { - prevNs = foundNs; - } else { - assert prevNs == foundNs : "All correlation variables should resolve" + " to the same namespace." + " Prev ns=" + prevNs + ", new ns=" + foundNs; - } + // implement SqlVisitor + public RexNode visit(SqlDataTypeSpec type) { + throw new UnsupportedOperationException(); + } - int namespaceOffset = 0; - if (childNamespaceIndex > 0) { - // If not the first child, need to figure out the width - // of output types from all the preceding namespaces - assert ancestorScope instanceof ListScope; - List children = ((ListScope) ancestorScope).getChildren(); - - for (int i = 0; i < childNamespaceIndex; i++) { - SqlValidatorNamespace child = children.get(i); - namespaceOffset += child.getRowType().getFieldCount(); - } - } + // implement SqlVisitor + public RexNode visit(SqlDynamicParam param) { + return convertDynamicParam(param); + } - RelDataTypeField field = catalogReader.field(foundNs.getRowType(), originalFieldName); - int pos = namespaceOffset + field.getIndex(); + // implement SqlVisitor + public RexNode visit(SqlIntervalQualifier intervalQualifier) { + return convertInterval(intervalQualifier); + } - assert field.getType() == lookup.getFieldAccess(correlName).getField().getType(); + public List getColumnMonotonicities() { + return columnMonotonicities; + } - assert pos != -1; + } - if (bb.mapRootRelToFieldProjection.containsKey(bb.root)) { - // bb.root is an aggregate and only projects group by - // keys. - Map exprProjection = bb.mapRootRelToFieldProjection.get(bb.root); + /** Deferred lookup. */ + private static class DeferredLookup { + Blackboard bb; + String originalRelName; - // subquery can reference group by keys projected from - // the root of the outer relation. - if (exprProjection.containsKey(pos)) { - pos = exprProjection.get(pos); - } else { - // correl not grouped - throw Util.newInternal("Identifier '" + originalRelName + "." + originalFieldName + "' is not a group expr"); - } - } + DeferredLookup( + Blackboard bb, + String originalRelName) { + this.bb = bb; + this.originalRelName = originalRelName; + } - requiredColumns.set(pos); - correlNames.add(correlName); - } + public RexFieldAccess getFieldAccess(CorrelationId name) { + return (RexFieldAccess) bb.mapCorrelateToRex.get(name); + } - if (!correlNames.isEmpty()) { - if (correlNames.size() > 1) { - // The same table was referenced more than once. - // So we deduplicate - RelShuttle dedup = new DeduplicateCorrelateVariables(rexBuilder, correlNames.get(0), ImmutableSet.copyOf(Util.skip(correlNames))); - rightRel = rightRel.accept(dedup); - } - LogicalCorrelate corr = LogicalCorrelate.create(leftRel, rightRel, new CorrelationId(correlNames.get(0)), requiredColumns.build(), SemiJoinType.of(joinType)); - if (!joinCond.isAlwaysTrue()) { - return RelOptUtil.createFilter(corr, joinCond); - } - return corr; - } - } + public String getOriginalRelName() { + return originalRelName; + } + } + + /** + * An implementation of DefaultValueFactory which always supplies NULL. + */ + class NullDefaultValueFactory implements DefaultValueFactory { + public boolean isGeneratedAlways( + RelOptTable table, + int iColumn) { + return false; + } - final Join originalJoin = (Join) RelFactories.DEFAULT_JOIN_FACTORY.createJoin(leftRel, rightRel, joinCond, joinType, ImmutableSet. of(), false); + public RexNode newColumnDefaultValue( + RelOptTable table, + int iColumn) { + return rexBuilder.constantNull(); + } - return RelOptUtil.pushDownJoinConditions(originalJoin); + public RexNode newAttributeInitializer( + RelDataType type, + SqlFunction constructor, + int iAttribute, + List constructorArgs) { + return rexBuilder.constantNull(); + } + } + + /** + * A default implementation of SubqueryConverter that does no conversion. + */ + private class NoOpSubqueryConverter implements SubqueryConverter { + // implement SubqueryConverter + public boolean canConvertSubquery() { + return false; } - private static boolean containsGet(RexNode node) { - try { - node.accept(new RexVisitorImpl(true) { - @Override - public Void visitCall(RexCall call) { - if (call.getOperator() == RexBuilder.GET_OPERATOR) { - throw Util.FoundOne.NULL; - } - return super.visitCall(call); - } - }); - return false; - } catch (Util.FoundOne e) { - return true; - } + // implement SubqueryConverter + public RexNode convertSubquery( + SqlCall subquery, + SqlToRelConverter parentConverter, + boolean isExists, + boolean isExplain) { + throw new IllegalArgumentException(); } + } + + /** + * Converts expressions to aggregates. + * + *

Consider the expression + * + *

+ * {@code SELECT deptno, SUM(2 * sal) FROM emp GROUP BY deptno} + *
+ * + *

Then: + * + *

    + *
  • groupExprs = {SqlIdentifier(deptno)}
    • + *
  • convertedInputExprs = {RexInputRef(deptno), 2 * + * RefInputRef(sal)}
    • + *
  • inputRefs = {RefInputRef(#0), RexInputRef(#1)}
    • + *
  • aggCalls = {AggCall(SUM, {1})}
    • + *
+ */ + protected class AggConverter implements SqlVisitor { + private final Blackboard bb; + public final AggregatingSelectScope aggregatingSelectScope; + + private final Map nameMap = Maps.newHashMap(); /** - * Determines whether a subquery is non-correlated. Note that a - * non-correlated subquery can contain correlated references, provided those - * references do not reference select statements that are parents of the - * subquery. - * - * @param subq the subquery - * @param bb blackboard used while converting the subquery, i.e., the - * blackboard of the parent query of this subquery - * @return true if the subquery is non-correlated. + * The group-by expressions, in {@link SqlNode} format. */ - private boolean isSubQueryNonCorrelated(RelNode subq, Blackboard bb) { - Set correlatedVariables = RelOptUtil.getVariablesUsed(subq); - for (String correlName : correlatedVariables) { - DeferredLookup lookup = mapCorrelToDeferred.get(correlName); - String originalRelName = lookup.getOriginalRelName(); - - int[] nsIndexes = { -1 }; - final SqlValidatorScope[] ancestorScopes = { null }; - SqlValidatorNamespace foundNs = lookup.bb.scope.resolve(ImmutableList.of(originalRelName), ancestorScopes, nsIndexes); - - assert foundNs != null; - assert nsIndexes.length == 1; - - SqlValidatorScope ancestorScope = ancestorScopes[0]; - - // If the correlated reference is in a scope that's "above" the - // subquery, then this is a correlated subquery. - SqlValidatorScope parentScope = bb.scope; - do { - if (ancestorScope == parentScope) { - return false; - } - if (parentScope instanceof DelegatingScope) { - parentScope = ((DelegatingScope) parentScope).getParent(); - } else { - break; - } - } while (parentScope != null); - } - return true; - } + private final SqlNodeList groupExprs = + new SqlNodeList(SqlParserPos.ZERO); /** - * Returns a list of fields to be prefixed to each relational expression. - * - * @return List of system fields + * Input expressions for the group columns and aggregates, in + * {@link RexNode} format. The first elements of the list correspond to the + * elements in {@link #groupExprs}; the remaining elements are for + * aggregates. */ - protected List getSystemFields() { - return Collections.emptyList(); - } - - private RexNode convertJoinCondition(Blackboard bb, SqlValidatorNamespace leftNamespace, SqlValidatorNamespace rightNamespace, SqlNode condition, JoinConditionType conditionType, RelNode leftRel, RelNode rightRel) { - if (condition == null) { - return rexBuilder.makeLiteral(true); - } - bb.setRoot(ImmutableList.of(leftRel, rightRel)); - replaceSubqueries(bb, condition, RelOptUtil.Logic.UNKNOWN_AS_FALSE); - switch (conditionType) { - case ON: - bb.setRoot(ImmutableList.of(leftRel, rightRel)); - return bb.convertExpression(condition); - case USING: - final SqlNodeList list = (SqlNodeList) condition; - final List nameList = new ArrayList<>(); - for (SqlNode columnName : list) { - final SqlIdentifier id = (SqlIdentifier) columnName; - String name = id.getSimple(); - nameList.add(name); - } - return convertUsing(leftNamespace, rightNamespace, nameList); - default: - throw Util.unexpected(conditionType); - } - } + private final List convertedInputExprs = Lists.newArrayList(); /** - * Returns an expression for matching columns of a USING clause or inferred - * from NATURAL JOIN. "a JOIN b USING (x, y)" becomes "a.x = b.x AND a.y = - * b.y". Returns null if the column list is empty. - * - * @param leftNamespace Namespace of left input to join - * @param rightNamespace Namespace of right input to join - * @param nameList List of column names to join on - * @return Expression to match columns from name list, or true if name list - * is empty + * Names of {@link #convertedInputExprs}, where the expressions are + * simple mappings to input fields. */ - private RexNode convertUsing(SqlValidatorNamespace leftNamespace, SqlValidatorNamespace rightNamespace, List nameList) { - final List list = Lists.newArrayList(); - for (String name : nameList) { - List operands = new ArrayList<>(); - int offset = 0; - for (SqlValidatorNamespace n : ImmutableList.of(leftNamespace, rightNamespace)) { - final RelDataType rowType = n.getRowType(); - final RelDataTypeField field = catalogReader.field(rowType, name); - operands.add(rexBuilder.makeInputRef(field.getType(), offset + field.getIndex())); - offset += rowType.getFieldList().size(); - } - list.add(rexBuilder.makeCall(SqlStdOperatorTable.EQUALS, operands)); - } - return RexUtil.composeConjunction(rexBuilder, list, false); - } - - private static JoinRelType convertJoinType(JoinType joinType) { - switch (joinType) { - case COMMA: - case INNER: - case CROSS: - return JoinRelType.INNER; - case FULL: - return JoinRelType.FULL; - case LEFT: - return JoinRelType.LEFT; - case RIGHT: - return JoinRelType.RIGHT; - default: - throw Util.unexpected(joinType); - } - } + private final List convertedInputExprNames = Lists.newArrayList(); + + private final List aggCalls = Lists.newArrayList(); + private final Map aggMapping = Maps.newHashMap(); + private final Map aggCallMapping = + Maps.newHashMap(); /** - * Converts the SELECT, GROUP BY and HAVING clauses of an aggregate query. + * Creates an AggConverter. * - *

This method extracts SELECT, GROUP BY and HAVING clauses, and creates - * an {@link AggConverter}, then delegates to {@link #createAggImpl}. - * Derived class may override this method to change any of those clauses or - * specify a different {@link AggConverter}. + *

The select parameter provides enough context to name + * aggregate calls which are top-level select list items. * - * @param bb Scope within which to resolve identifiers - * @param select Query - * @param orderExprList Additional expressions needed to implement ORDER BY + * @param bb Blackboard + * @param select Query being translated; provides context to give */ - protected void convertAgg(Blackboard bb, SqlSelect select, List orderExprList) { - assert bb.root != null : "precondition: child != null"; - SqlNodeList groupList = select.getGroup(); - SqlNodeList selectList = select.getSelectList(); - SqlNode having = select.getHaving(); - - final AggConverter aggConverter = new AggConverter(bb, select); - createAggImpl(bb, aggConverter, selectList, groupList, having, orderExprList); - } - - protected final void createAggImpl(Blackboard bb, final AggConverter aggConverter, SqlNodeList selectList, SqlNodeList groupList, SqlNode having, List orderExprList) { - // Find aggregate functions in SELECT and HAVING clause - final AggregateFinder aggregateFinder = new AggregateFinder(); - selectList.accept(aggregateFinder); - if (having != null) { - having.accept(aggregateFinder); - } - - // first replace the subqueries inside the aggregates - // because they will provide input rows to the aggregates. - replaceSubqueries(bb, aggregateFinder.list, RelOptUtil.Logic.TRUE_FALSE_UNKNOWN); - - // If group-by clause is missing, pretend that it has zero elements. - if (groupList == null) { - groupList = SqlNodeList.EMPTY; - } - - replaceSubqueries(bb, groupList, RelOptUtil.Logic.TRUE_FALSE_UNKNOWN); - - // register the group exprs - - // build a map to remember the projections from the top scope to the - // output of the current root. - // - // Currently farrago allows expressions, not just column references in - // group by list. This is not SQL 2003 compliant. - - final AggregatingSelectScope scope = aggConverter.aggregatingSelectScope; - for (SqlNode groupExpr : scope.groupExprList) { - aggConverter.addGroupExpr(groupExpr); - } - - RexNode havingExpr = null; - final List> projects = Lists.newArrayList(); - - try { - Util.permAssert(bb.agg == null, "already in agg mode"); - bb.agg = aggConverter; - - // convert the select and having expressions, so that the - // agg converter knows which aggregations are required - - selectList.accept(aggConverter); - for (SqlNode expr : orderExprList) { - expr.accept(aggConverter); - } - if (having != null) { - having.accept(aggConverter); - } + public AggConverter(Blackboard bb, SqlSelect select) { + this.bb = bb; + this.aggregatingSelectScope = + (AggregatingSelectScope) bb.getValidator().getSelectScope(select); + + // Collect all expressions used in the select list so that aggregate + // calls can be named correctly. + final SqlNodeList selectList = select.getSelectList(); + for (int i = 0; i < selectList.size(); i++) { + SqlNode selectItem = selectList.get(i); + String name = null; + if (SqlUtil.isCallTo( + selectItem, + SqlStdOperatorTable.AS)) { + final SqlCall call = (SqlCall) selectItem; + selectItem = call.operand(0); + name = call.operand(1).toString(); + } + if (name == null) { + name = validator.deriveAlias(selectItem, i); + } + nameMap.put(selectItem.toString(), name); + } + } - // compute inputs to the aggregator - List preExprs = aggConverter.getPreExprs(); - List preNames = aggConverter.getPreNames(); + public int addGroupExpr(SqlNode expr) { + RexNode convExpr = bb.convertExpression(expr); + int ref = lookupGroupExpr(expr); + if (ref >= 0) { + return ref; + } + final int index = groupExprs.size(); + groupExprs.add(expr); + String name = nameMap.get(expr.toString()); + addExpr(convExpr, name); + return index; + } - if (preExprs.size() == 0) { - // Special case for COUNT(*), where we can end up with no inputs - // at all. The rest of the system doesn't like 0-tuples, so we - // select a dummy constant here. - preExprs = Collections.singletonList((RexNode) rexBuilder.makeExactLiteral(BigDecimal.ZERO)); - preNames = Collections.singletonList(null); - } + /** + * Adds an expression, deducing an appropriate name if possible. + * + * @param expr Expression + * @param name Suggested name + */ + private void addExpr(RexNode expr, String name) { + convertedInputExprs.add(expr); + if ((name == null) && (expr instanceof RexInputRef)) { + final int i = ((RexInputRef) expr).getIndex(); + name = bb.root.getRowType().getFieldList().get(i).getName(); + } + if (convertedInputExprNames.contains(name)) { + // In case like 'SELECT ... GROUP BY x, y, x', don't add + // name 'x' twice. + name = null; + } + convertedInputExprNames.add(name); + } - final RelNode inputRel = bb.root; + // implement SqlVisitor + public Void visit(SqlIdentifier id) { + return null; + } - // Project the expressions required by agg and having. - bb.setRoot(RelOptUtil.createProject(inputRel, preExprs, preNames, true), false); - bb.mapRootRelToFieldProjection.put(bb.root, scope.groupExprProjection); + // implement SqlVisitor + public Void visit(SqlNodeList nodeList) { + for (int i = 0; i < nodeList.size(); i++) { + nodeList.get(i).accept(this); + } + return null; + } - // REVIEW jvs 31-Oct-2007: doesn't the declaration of - // monotonicity here assume sort-based aggregation at - // the physical level? + // implement SqlVisitor + public Void visit(SqlLiteral lit) { + return null; + } - // Tell bb which of group columns are sorted. - bb.columnMonotonicities.clear(); - for (SqlNode groupItem : groupList) { - bb.columnMonotonicities.add(bb.scope.getMonotonicity(groupItem)); - } + // implement SqlVisitor + public Void visit(SqlDataTypeSpec type) { + return null; + } - // Add the aggregator - bb.setRoot(createAggregate(bb, aggConverter.aggregatingSelectScope.indicator, scope.groupSet, scope.groupSets, aggConverter.getAggCalls()), false); - - // Generate NULL values for rolled-up not-null fields. - final Aggregate aggregate = (Aggregate) bb.root; - if (aggregate.getGroupType() != Aggregate.Group.SIMPLE) { - assert aggregate.indicator; - List> projects2 = Lists.newArrayList(); - int converted = 0; - final int groupCount = aggregate.getGroupSet().cardinality(); - for (RelDataTypeField field : aggregate.getRowType().getFieldList()) { - final int i = field.getIndex(); - final RexNode rex; - if (i < groupCount && scope.isNullable(i)) { - ++converted; - - rex = rexBuilder.makeCall(SqlStdOperatorTable.CASE, rexBuilder.makeInputRef(aggregate, groupCount + i), rexBuilder.makeCast(typeFactory.createTypeWithNullability(field.getType(), true), rexBuilder.constantNull()), rexBuilder.makeInputRef(aggregate, i)); - } else { - rex = rexBuilder.makeInputRef(aggregate, i); - } - projects2.add(Pair.of(rex, field.getName())); - } - if (converted > 0) { - bb.setRoot(RelOptUtil.createProject(bb.root, projects2, true), false); - } - } + // implement SqlVisitor + public Void visit(SqlDynamicParam param) { + return null; + } - bb.mapRootRelToFieldProjection.put(bb.root, scope.groupExprProjection); + // implement SqlVisitor + public Void visit(SqlIntervalQualifier intervalQualifier) { + return null; + } - // Replace subqueries in having here and modify having to use - // the replaced expressions - if (having != null) { - SqlNode newHaving = pushDownNotForIn(having); - replaceSubqueries(bb, newHaving, RelOptUtil.Logic.UNKNOWN_AS_FALSE); - havingExpr = bb.convertExpression(newHaving); - if (havingExpr.isAlwaysTrue()) { - havingExpr = null; - } - } + public Void visit(SqlCall call) { + switch (call.getKind()) { + case FILTER: + translateAgg((SqlCall) call.operand(0), call.operand(1), call); + return null; + case SELECT: + // rchen 2006-10-17: + // for now do not detect aggregates in subqueries. + return null; + } + // ignore window aggregates and ranking functions (associated with OVER operator) + if (call.getOperator().getKind() == SqlKind.OVER) { + return null; + } + if (call.getOperator().isAggregator()) { + translateAgg(call, null, call); + return null; + } + for (SqlNode operand : call.getOperandList()) { + // Operands are occasionally null, e.g. switched CASE arg 0. + if (operand != null) { + operand.accept(this); + } + } + return null; + } - // Now convert the other subqueries in the select list. - // This needs to be done separately from the subquery inside - // any aggregate in the select list, and after the aggregate rel - // is allocated. - replaceSubqueries(bb, selectList, RelOptUtil.Logic.TRUE_FALSE_UNKNOWN); - - // Now subqueries in the entire select list have been converted. - // Convert the select expressions to get the final list to be - // projected. - int k = 0; - - // For select expressions, use the field names previously assigned - // by the validator. If we derive afresh, we might generate names - // like "EXPR$2" that don't match the names generated by the - // validator. This is especially the case when there are system - // fields; system fields appear in the relnode's rowtype but do not - // (yet) appear in the validator type. - final SelectScope selectScope = SqlValidatorUtil.getEnclosingSelectScope(bb.scope); - assert selectScope != null; - final SqlValidatorNamespace selectNamespace = validator.getNamespace(selectScope.getNode()); - final List names = selectNamespace.getRowType().getFieldNames(); - int sysFieldCount = selectList.size() - names.size(); - for (SqlNode expr : selectList) { - projects.add(Pair.of(bb.convertExpression(expr), k < sysFieldCount ? validator.deriveAlias(expr, k++) : names.get(k++ - sysFieldCount))); + private void translateAgg(SqlCall call, SqlNode filter, SqlCall outerCall) { + assert bb.agg == this; + final List args = new ArrayList<>(); + int filterArg = -1; + final List argTypes = + call.getOperator() instanceof SqlCountAggFunction + ? new ArrayList(call.getOperandList().size()) + : null; + try { + // switch out of agg mode + bb.agg = null; + for (SqlNode operand : call.getOperandList()) { + + // special case for COUNT(*): delete the * + if (operand instanceof SqlIdentifier) { + SqlIdentifier id = (SqlIdentifier) operand; + if (id.isStar() || isSimpleCount(call)) { /* OVERRIDE POINT */ + assert call.operandCount() == 1; + assert args.isEmpty(); + break; } + } + RexNode convertedExpr = bb.convertExpression(operand); + assert convertedExpr != null; + if (argTypes != null) { + argTypes.add(convertedExpr.getType()); + } + args.add(lookupOrCreateGroupExpr(convertedExpr)); + } + + if (filter != null) { + RexNode convertedExpr = bb.convertExpression(filter); + assert convertedExpr != null; + if (convertedExpr.getType().isNullable()) { + convertedExpr = + rexBuilder.makeCall(SqlStdOperatorTable.IS_TRUE, convertedExpr); + } + filterArg = lookupOrCreateGroupExpr(convertedExpr); + } + } finally { + // switch back into agg mode + bb.agg = this; + } + + final SqlAggFunction aggFunction = + (SqlAggFunction) call.getOperator(); + RelDataType type = validator.deriveType(bb.scope, call); + boolean distinct = false; + SqlLiteral quantifier = call.getFunctionQuantifier(); + if ((null != quantifier) + && (quantifier.getValue() == SqlSelectKeyword.DISTINCT)) { + distinct = true; + } + final AggregateCall aggCall = + AggregateCall.create( + aggFunction, + distinct, + args, + filterArg, + type, + nameMap.get(outerCall.toString())); + final AggregatingSelectScope.Resolved r = + aggregatingSelectScope.resolved.get(); + RexNode rex = + rexBuilder.addAggCall( + aggCall, + groupExprs.size(), + r.indicator, + aggCalls, + aggCallMapping, + argTypes); + aggMapping.put(outerCall, rex); + } - for (SqlNode expr : orderExprList) { - projects.add(Pair.of(bb.convertExpression(expr), validator.deriveAlias(expr, k++))); + /* OVERRIDE POINT */ + private boolean isSimpleCount(SqlCall call) { + if (call.getOperator().isName("COUNT") && call.operandCount() == 1) { + final SqlNode parm = call.operand(0); + if ((parm instanceof SqlIdentifier || parm instanceof SqlNumericLiteral) // + && call.getFunctionQuantifier() == null) { + return true; } - } finally { - bb.agg = null; } + return false; + } - // implement HAVING (we have already checked that it is non-trivial) - if (havingExpr != null) { - bb.setRoot(RelOptUtil.createFilter(bb.root, havingExpr), false); + private int lookupOrCreateGroupExpr(RexNode expr) { + for (int i = 0; i < convertedInputExprs.size(); i++) { + RexNode convertedInputExpr = convertedInputExprs.get(i); + if (expr.toString().equals(convertedInputExpr.toString())) { + return i; } + } - // implement the SELECT list - bb.setRoot(RelOptUtil.createProject(bb.root, projects, true), false); - - // Tell bb which of group columns are sorted. - bb.columnMonotonicities.clear(); - for (SqlNode selectItem : selectList) { - bb.columnMonotonicities.add(bb.scope.getMonotonicity(selectItem)); - } + // not found -- add it + int index = convertedInputExprs.size(); + addExpr(expr, null); + return index; } /** - * Creates an Aggregate. - * - *

In case the aggregate rel changes the order in which it projects - * fields, the groupExprProjection parameter is provided, and - * the implementation of this method may modify it. - * - *

The sortedCount parameter is the number of expressions - * known to be monotonic. These expressions must be on the leading edge of - * the grouping keys. The default implementation of this method ignores this - * parameter. - * - * @param bb Blackboard - * @param indicator Whether to output fields indicating grouping sets - * @param groupSet Bit set of ordinals of grouping columns - * @param groupSets Grouping sets - * @param aggCalls Array of calls to aggregate functions - * @return LogicalAggregate + * If an expression is structurally identical to one of the group-by + * expressions, returns a reference to the expression, otherwise returns + * null. */ - protected RelNode createAggregate(Blackboard bb, boolean indicator, ImmutableBitSet groupSet, ImmutableList groupSets, List aggCalls) { - return LogicalAggregate.create(bb.root, indicator, groupSet, groupSets, aggCalls); + public int lookupGroupExpr(SqlNode expr) { + for (int i = 0; i < groupExprs.size(); i++) { + SqlNode groupExpr = groupExprs.get(i); + if (expr.equalsDeep(groupExpr, false)) { + return i; + } + } + return -1; } - public RexDynamicParam convertDynamicParam(final SqlDynamicParam dynamicParam) { - // REVIEW jvs 8-Jan-2005: dynamic params may be encountered out of - // order. Should probably cross-check with the count from the parser - // at the end and make sure they all got filled in. Why doesn't List - // have a resize() method?!? Make this a utility. - while (dynamicParam.getIndex() >= dynamicParamSqlNodes.size()) { - dynamicParamSqlNodes.add(null); - } - - dynamicParamSqlNodes.set(dynamicParam.getIndex(), dynamicParam); - return rexBuilder.makeDynamicParam(getDynamicParamType(dynamicParam.getIndex()), dynamicParam.getIndex()); + public RexNode lookupAggregates(SqlCall call) { + // assert call.getOperator().isAggregator(); + assert bb.agg == this; + + switch (call.getKind()) { + case GROUPING: + case GROUPING_ID: + case GROUP_ID: + final RelDataType type = validator.getValidatedNodeType(call); + if (!aggregatingSelectScope.resolved.get().indicator) { + return rexBuilder.makeExactLiteral( + TWO.pow(effectiveArgCount(call)).subtract(BigDecimal.ONE), type); + } else { + final List operands; + switch (call.getKind()) { + case GROUP_ID: + operands = ImmutableIntList.range(0, groupExprs.size()); + break; + default: + operands = Lists.newArrayList(); + for (SqlNode operand : call.getOperandList()) { + final int x = lookupGroupExpr(operand); + assert x >= 0; + operands.add(x); + } + } + RexNode node = null; + int shift = operands.size(); + for (int operand : operands) { + node = bitValue(node, type, operand, --shift); + } + return node; + } + } + return aggMapping.get(call); } - /** - * Creates a list of collations required to implement the ORDER BY clause, - * if there is one. Populates extraOrderExprs with any sort - * expressions which are not in the select clause. - * - * @param bb Scope within which to resolve identifiers - * @param select Select clause. Never null, because we invent a - * dummy SELECT if ORDER BY is applied to a set - * operation (UNION etc.) - * @param orderList Order by clause, may be null - * @param extraOrderExprs Sort expressions which are not in the select - * clause (output) - * @param collationList List of collations (output) - */ - protected void gatherOrderExprs(Blackboard bb, SqlSelect select, SqlNodeList orderList, List extraOrderExprs, List collationList) { - // TODO: add validation rules to SqlValidator also - assert bb.root != null : "precondition: child != null"; - assert select != null; - if (orderList == null) { - return; - } - for (SqlNode orderItem : orderList) { - collationList.add(convertOrderItem(select, orderItem, extraOrderExprs, RelFieldCollation.Direction.ASCENDING, RelFieldCollation.NullDirection.UNSPECIFIED)); - } + private int effectiveArgCount(SqlCall call) { + switch (call.getKind()) { + case GROUPING: + return 1; + case GROUPING_ID: + return call.operandCount(); + case GROUP_ID: + return groupExprs.size(); + default: + throw new AssertionError(call.getKind()); + } } - protected RelFieldCollation convertOrderItem(SqlSelect select, SqlNode orderItem, List extraExprs, RelFieldCollation.Direction direction, RelFieldCollation.NullDirection nullDirection) { - assert select != null; - // Handle DESC keyword, e.g. 'select a, b from t order by a desc'. - switch (orderItem.getKind()) { - case DESCENDING: - return convertOrderItem(select, ((SqlCall) orderItem).operand(0), extraExprs, RelFieldCollation.Direction.DESCENDING, nullDirection); - case NULLS_FIRST: - return convertOrderItem(select, ((SqlCall) orderItem).operand(0), extraExprs, direction, RelFieldCollation.NullDirection.FIRST); - case NULLS_LAST: - return convertOrderItem(select, ((SqlCall) orderItem).operand(0), extraExprs, direction, RelFieldCollation.NullDirection.LAST); - } - - SqlNode converted = validator.expandOrderExpr(select, orderItem); - - // Scan the select list and order exprs for an identical expression. - final SelectScope selectScope = validator.getRawSelectScope(select); - int ordinal = -1; - for (SqlNode selectItem : selectScope.getExpandedSelectList()) { - ++ordinal; - if (converted.equalsDeep(stripAs(selectItem), false)) { - return new RelFieldCollation(ordinal, direction, nullDirection); - } - } - - for (SqlNode extraExpr : extraExprs) { - ++ordinal; - if (converted.equalsDeep(extraExpr, false)) { - return new RelFieldCollation(ordinal, direction, nullDirection); - } - } + private RexNode bitValue(RexNode previous, RelDataType type, int x, + int shift) { + final AggregatingSelectScope.Resolved r = + aggregatingSelectScope.resolved.get(); + RexNode node = rexBuilder.makeCall(SqlStdOperatorTable.CASE, + rexBuilder.makeInputRef(bb.root, r.groupExprList.size() + x), + rexBuilder.makeExactLiteral(BigDecimal.ONE, type), + rexBuilder.makeExactLiteral(BigDecimal.ZERO, type)); + if (shift > 0) { + node = rexBuilder.makeCall(SqlStdOperatorTable.MULTIPLY, node, + rexBuilder.makeExactLiteral(TWO.pow(shift), type)); + } + if (previous != null) { + node = rexBuilder.makeCall(SqlStdOperatorTable.PLUS, previous, node); + } + return node; + } - // TODO: handle collation sequence - // TODO: flag expressions as non-standard + public List getPreExprs() { + return convertedInputExprs; + } - extraExprs.add(converted); - return new RelFieldCollation(ordinal + 1, direction, nullDirection); + public List getPreNames() { + return convertedInputExprNames; } - protected boolean enableDecorrelation() { - // disable subquery decorrelation when needed. - // e.g. if outer joins are not supported. - return decorrelationEnabled; + public List getAggCalls() { + return aggCalls; } - protected RelNode decorrelateQuery(RelNode rootRel) { - return RelDecorrelator.decorrelateQuery(rootRel); + public RelDataTypeFactory getTypeFactory() { + return typeFactory; } + } + + /** + * Context to find a relational expression to a field offset. + */ + private static class LookupContext { + private final List> relOffsetList = + new ArrayList<>(); /** - * Sets whether to trim unused fields as part of the conversion process. + * Creates a LookupContext with multiple input relational expressions. * - * @param trim Whether to trim unused fields + * @param bb Context for translating this subquery + * @param rels Relational expressions + * @param systemFieldCount Number of system fields */ - public void setTrimUnusedFields(boolean trim) { - this.trimUnusedFields = trim; + LookupContext(Blackboard bb, List rels, int systemFieldCount) { + bb.flatten(rels, systemFieldCount, new int[]{0}, relOffsetList); } /** - * Returns whether to trim unused fields as part of the conversion process. + * Returns the relational expression with a given offset, and the + * ordinal in the combined row of its first field. + * + *

For example, in {@code Emp JOIN Dept}, findRel(1) returns the + * relational expression for {@code Dept} and offset 6 (because + * {@code Emp} has 6 fields, therefore the first field of {@code Dept} + * is field 6. * - * @return Whether to trim unused fields + * @param offset Offset of relational expression in FROM clause + * @return Relational expression and the ordinal of its first field */ - public boolean isTrimUnusedFields() { - /* OVERRIDE POINT */ - return false; + Pair findRel(int offset) { + return relOffsetList.get(offset); } - + } + + /** + * Shuttle which walks over a tree of {@link RexNode}s and applies 'over' to + * all agg functions. + * + *

This is necessary because the returned expression is not necessarily a + * call to an agg function. For example, + * + *

AVG(x)
+ * + * becomes + * + *
SUM(x) / COUNT(x)
+ * + *

Any aggregate functions are converted to calls to the internal + * $Histogram aggregation function and accessors such as + * $HistogramMin; for example, + * + *

MIN(x), MAX(x)
+ * + * are converted to + * + *
$HistogramMin($Histogram(x)), + * $HistogramMax($Histogram(x))
+ * + * Common sub-expression elmination will ensure that only one histogram is + * computed. + */ + private class HistogramShuttle extends RexShuttle { /** - * Recursively converts a query to a relational expression. - * - * @param query Query - * @param top Whether this query is the top-level query of the - * statement - * @param targetRowType Target row type, or null - * @return Relational expression + * Whether to convert calls to MIN(x) to HISTOGRAM_MIN(HISTOGRAM(x)). + * Histograms allow rolling computation, but require more space. */ - protected RelNode convertQueryRecursive(SqlNode query, boolean top, RelDataType targetRowType) { - switch (query.getKind()) { - case SELECT: - return convertSelect((SqlSelect) query); - case INSERT: - return convertInsert((SqlInsert) query); - case DELETE: - return convertDelete((SqlDelete) query); - case UPDATE: - return convertUpdate((SqlUpdate) query); - case MERGE: - return convertMerge((SqlMerge) query); - case UNION: - case INTERSECT: - case EXCEPT: - return convertSetOp((SqlCall) query); - case WITH: - return convertWith((SqlWith) query); - case VALUES: - return convertValues((SqlCall) query, targetRowType); - default: - throw Util.newInternal("not a query: " + query); - } + static final boolean ENABLE_HISTOGRAM_AGG = false; + + private final List partitionKeys; + private final ImmutableList orderKeys; + private final RexWindowBound lowerBound; + private final RexWindowBound upperBound; + private final SqlWindow window; + + HistogramShuttle( + List partitionKeys, + ImmutableList orderKeys, + RexWindowBound lowerBound, RexWindowBound upperBound, + SqlWindow window) { + this.partitionKeys = partitionKeys; + this.orderKeys = orderKeys; + this.lowerBound = lowerBound; + this.upperBound = upperBound; + this.window = window; + } + + public RexNode visitCall(RexCall call) { + final SqlOperator op = call.getOperator(); + if (!(op instanceof SqlAggFunction)) { + return super.visitCall(call); + } + final SqlAggFunction aggOp = (SqlAggFunction) op; + final RelDataType type = call.getType(); + List exprs = call.getOperands(); + + SqlFunction histogramOp = !ENABLE_HISTOGRAM_AGG + ? null + : getHistogramOp(aggOp); + + if (histogramOp != null) { + final RelDataType histogramType = computeHistogramType(type); + + // For DECIMAL, since it's already represented as a bigint we + // want to do a reinterpretCast instead of a cast to avoid + // losing any precision. + boolean reinterpretCast = + type.getSqlTypeName() == SqlTypeName.DECIMAL; + + // Replace original expression with CAST of not one + // of the supported types + if (histogramType != type) { + exprs = new ArrayList<>(exprs); + exprs.set( + 0, + reinterpretCast + ? rexBuilder.makeReinterpretCast(histogramType, exprs.get(0), + rexBuilder.makeLiteral(false)) + : rexBuilder.makeCast(histogramType, exprs.get(0))); + } + + RexCallBinding bind = + new RexCallBinding( + rexBuilder.getTypeFactory(), + SqlStdOperatorTable.HISTOGRAM_AGG, + exprs, + ImmutableList.of()); + + RexNode over = + rexBuilder.makeOver( + SqlStdOperatorTable.HISTOGRAM_AGG + .inferReturnType(bind), + SqlStdOperatorTable.HISTOGRAM_AGG, + exprs, + partitionKeys, + orderKeys, + lowerBound, + upperBound, + window.isRows(), + window.isAllowPartial(), + false); + + RexNode histogramCall = + rexBuilder.makeCall( + histogramType, + histogramOp, + ImmutableList.of(over)); + + // If needed, post Cast result back to original + // type. + if (histogramType != type) { + if (reinterpretCast) { + histogramCall = + rexBuilder.makeReinterpretCast( + type, + histogramCall, + rexBuilder.makeLiteral(false)); + } else { + histogramCall = + rexBuilder.makeCast(type, histogramCall); + } + } + + return histogramCall; + } else { + boolean needSum0 = aggOp == SqlStdOperatorTable.SUM + && type.isNullable(); + SqlAggFunction aggOpToUse = + needSum0 ? SqlStdOperatorTable.SUM0 + : aggOp; + return rexBuilder.makeOver( + type, + aggOpToUse, + exprs, + partitionKeys, + orderKeys, + lowerBound, + upperBound, + window.isRows(), + window.isAllowPartial(), + needSum0); + } } /** - * Converts a set operation (UNION, INTERSECT, MINUS) into relational - * expressions. + * Returns the histogram operator corresponding to a given aggregate + * function. * - * @param call Call to set operator - * @return Relational expression - */ - protected RelNode convertSetOp(SqlCall call) { - final RelNode left = convertQueryRecursive(call.operand(0), false, null); - final RelNode right = convertQueryRecursive(call.operand(1), false, null); - boolean all = false; - if (call.getOperator() instanceof SqlSetOperator) { - all = ((SqlSetOperator) (call.getOperator())).isAll(); - } - switch (call.getKind()) { - case UNION: - return LogicalUnion.create(ImmutableList.of(left, right), all); - - case INTERSECT: - // TODO: all - if (!all) { - return LogicalIntersect.create(ImmutableList.of(left, right), all); - } else { - throw Util.newInternal("set operator INTERSECT ALL not suported"); - } - - case EXCEPT: - // TODO: all - if (!all) { - return LogicalMinus.create(ImmutableList.of(left, right), all); - } else { - throw Util.newInternal("set operator EXCEPT ALL not suported"); - } - - default: - throw Util.unexpected(call.getKind()); - } - } - - protected RelNode convertInsert(SqlInsert call) { - RelOptTable targetTable = getTargetTable(call); - - final RelDataType targetRowType = validator.getValidatedNodeType(call); - assert targetRowType != null; - RelNode sourceRel = convertQueryRecursive(call.getSource(), false, targetRowType); - RelNode massagedRel = convertColumnList(call, sourceRel); - - return createModify(targetTable, massagedRel); - } - - /** Creates a relational expression to modify a table or modifiable view. */ - private RelNode createModify(RelOptTable targetTable, RelNode source) { - final ModifiableTable modifiableTable = targetTable.unwrap(ModifiableTable.class); - if (modifiableTable != null) { - return modifiableTable.toModificationRel(cluster, targetTable, catalogReader, source, LogicalTableModify.Operation.INSERT, null, false); - } - final ModifiableView modifiableView = targetTable.unwrap(ModifiableView.class); - if (modifiableView != null) { - final Table delegateTable = modifiableView.getTable(); - final RelDataType delegateRowType = delegateTable.getRowType(typeFactory); - final RelOptTable delegateRelOptTable = RelOptTableImpl.create(null, delegateRowType, delegateTable, modifiableView.getTablePath()); - final RelNode newSource = createSource(targetTable, source, modifiableView, delegateRowType); - return createModify(delegateRelOptTable, newSource); - } - return LogicalTableModify.create(targetTable, catalogReader, source, LogicalTableModify.Operation.INSERT, null, false); - } - - /** Wraps a relational expression in the projects and filters implied by - * a {@link ModifiableView}. - * - *

The input relational expression is suitable for inserting into the view, - * and the returned relational expression is suitable for inserting into its - * delegate table. - * - *

In principle, the delegate table of a view might be another modifiable - * view, and if so, the process can be repeated. */ - private RelNode createSource(RelOptTable targetTable, RelNode source, ModifiableView modifiableView, RelDataType delegateRowType) { - final ImmutableIntList mapping = modifiableView.getColumnMapping(); - assert mapping.size() == targetTable.getRowType().getFieldCount(); - - // For columns represented in the mapping, the expression is just a field - // reference. - final Map projectMap = new HashMap<>(); - final List filters = new ArrayList<>(); - for (int i = 0; i < mapping.size(); i++) { - int target = mapping.get(i); - if (target >= 0) { - projectMap.put(target, RexInputRef.of(i, source.getRowType())); - } - } - - // For columns that are not in the mapping, and have a constraint of the - // form "column = value", the expression is the literal "value". - // - // If a column has multiple constraints, the extra ones will become a - // filter. - final RexNode constraint = modifiableView.getConstraint(rexBuilder, delegateRowType); - RelOptUtil.inferViewPredicates(projectMap, filters, constraint); - final List> projects = new ArrayList<>(); - for (RelDataTypeField field : delegateRowType.getFieldList()) { - RexNode node = projectMap.get(field.getIndex()); - if (node == null) { - node = rexBuilder.makeNullLiteral(field.getType().getSqlTypeName()); - } - projects.add(Pair.of(rexBuilder.ensureType(field.getType(), node, false), field.getName())); - } - - source = RelOptUtil.createProject(source, projects, true); - if (filters.size() > 0) { - source = RelOptUtil.createFilter(source, filters); - } - return source; - } - - private RelOptTable.ToRelContext createToRelContext() { - return new RelOptTable.ToRelContext() { - public RelOptCluster getCluster() { - return cluster; - } - - public RelNode expandView(RelDataType rowType, String queryString, List schemaPath) { - return viewExpander.expandView(rowType, queryString, schemaPath); - } - }; - } - - public RelNode toRel(RelOptTable table) { - return table.toRel(createToRelContext()); - } - - protected RelOptTable getTargetTable(SqlNode call) { - SqlValidatorNamespace targetNs = validator.getNamespace(call).resolve(); - return SqlValidatorUtil.getRelOptTable(targetNs, catalogReader, null, null); - } - - /** - * Creates a source for an INSERT statement. - * - *

If the column list is not specified, source expressions match target - * columns in order. - * - *

If the column list is specified, Source expressions are mapped to - * target columns by name via targetColumnList, and may not cover the entire - * target table. So, we'll make up a full row, using a combination of - * default values and the source expressions provided. - * - * @param call Insert expression - * @param sourceRel Source relational expression - * @return Converted INSERT statement - */ - protected RelNode convertColumnList(SqlInsert call, RelNode sourceRel) { - RelDataType sourceRowType = sourceRel.getRowType(); - final RexNode sourceRef = rexBuilder.makeRangeReference(sourceRowType, 0, false); - final List targetColumnNames = new ArrayList<>(); - final List columnExprs = new ArrayList<>(); - collectInsertTargets(call, sourceRef, targetColumnNames, columnExprs); - - final RelOptTable targetTable = getTargetTable(call); - final RelDataType targetRowType = targetTable.getRowType(); - final List targetFields = targetRowType.getFieldList(); - final List sourceExps = new ArrayList<>(Collections. nCopies(targetFields.size(), null)); - final List fieldNames = new ArrayList<>(Collections. nCopies(targetFields.size(), null)); - - // Walk the name list and place the associated value in the - // expression list according to the ordinal value returned from - // the table construct, leaving nulls in the list for columns - // that are not referenced. - for (Pair p : Pair.zip(targetColumnNames, columnExprs)) { - RelDataTypeField field = catalogReader.field(targetRowType, p.left); - assert field != null : "column " + p.left + " not found"; - sourceExps.set(field.getIndex(), p.right); - } - - // Walk the expression list and get default values for any columns - // that were not supplied in the statement. Get field names too. - for (int i = 0; i < targetFields.size(); ++i) { - final RelDataTypeField field = targetFields.get(i); - final String fieldName = field.getName(); - fieldNames.set(i, fieldName); - if (sourceExps.get(i) != null) { - if (defaultValueFactory.isGeneratedAlways(targetTable, i)) { - throw RESOURCE.insertIntoAlwaysGenerated(fieldName).ex(); - } - continue; - } - sourceExps.set(i, defaultValueFactory.newColumnDefaultValue(targetTable, i)); - - // bare nulls are dangerous in the wrong hands - sourceExps.set(i, castNullLiteralIfNeeded(sourceExps.get(i), field.getType())); - } - - return RelOptUtil.createProject(sourceRel, sourceExps, fieldNames, true); - } - - private RexNode castNullLiteralIfNeeded(RexNode node, RelDataType type) { - if (!RexLiteral.isNullLiteral(node)) { - return node; - } - return rexBuilder.makeCast(type, node); - } - - /** - * Given an INSERT statement, collects the list of names to be populated and - * the expressions to put in them. - * - * @param call Insert statement - * @param sourceRef Expression representing a row from the source - * relational expression - * @param targetColumnNames List of target column names, to be populated - * @param columnExprs List of expressions, to be populated - */ - protected void collectInsertTargets(SqlInsert call, final RexNode sourceRef, final List targetColumnNames, List columnExprs) { - final RelOptTable targetTable = getTargetTable(call); - final RelDataType targetRowType = targetTable.getRowType(); - SqlNodeList targetColumnList = call.getTargetColumnList(); - if (targetColumnList == null) { - targetColumnNames.addAll(targetRowType.getFieldNames()); - } else { - for (int i = 0; i < targetColumnList.size(); i++) { - SqlIdentifier id = (SqlIdentifier) targetColumnList.get(i); - targetColumnNames.add(id.getSimple()); - } - } - - for (int i = 0; i < targetColumnNames.size(); i++) { - final RexNode expr = rexBuilder.makeFieldAccess(sourceRef, i); - columnExprs.add(expr); - } - } - - private RelNode convertDelete(SqlDelete call) { - RelOptTable targetTable = getTargetTable(call); - RelNode sourceRel = convertSelect(call.getSourceSelect()); - return LogicalTableModify.create(targetTable, catalogReader, sourceRel, LogicalTableModify.Operation.DELETE, null, false); - } - - private RelNode convertUpdate(SqlUpdate call) { - RelOptTable targetTable = getTargetTable(call); - - // convert update column list from SqlIdentifier to String - final List targetColumnNameList = new ArrayList<>(); - for (SqlNode node : call.getTargetColumnList()) { - SqlIdentifier id = (SqlIdentifier) node; - String name = id.getSimple(); - targetColumnNameList.add(name); - } - - RelNode sourceRel = convertSelect(call.getSourceSelect()); - - return LogicalTableModify.create(targetTable, catalogReader, sourceRel, LogicalTableModify.Operation.UPDATE, targetColumnNameList, false); - } - - private RelNode convertMerge(SqlMerge call) { - RelOptTable targetTable = getTargetTable(call); - - // convert update column list from SqlIdentifier to String - final List targetColumnNameList = new ArrayList<>(); - SqlUpdate updateCall = call.getUpdateCall(); - if (updateCall != null) { - for (SqlNode targetColumn : updateCall.getTargetColumnList()) { - SqlIdentifier id = (SqlIdentifier) targetColumn; - String name = id.getSimple(); - targetColumnNameList.add(name); - } - } - - // replace the projection of the source select with a - // projection that contains the following: - // 1) the expressions corresponding to the new insert row (if there is - // an insert) - // 2) all columns from the target table (if there is an update) - // 3) the set expressions in the update call (if there is an update) - - // first, convert the merge's source select to construct the columns - // from the target table and the set expressions in the update call - RelNode mergeSourceRel = convertSelect(call.getSourceSelect()); - - // then, convert the insert statement so we can get the insert - // values expressions - SqlInsert insertCall = call.getInsertCall(); - int nLevel1Exprs = 0; - List level1InsertExprs = null; - List level2InsertExprs = null; - if (insertCall != null) { - RelNode insertRel = convertInsert(insertCall); - - // if there are 2 level of projections in the insert source, combine - // them into a single project; level1 refers to the topmost project; - // the level1 projection contains references to the level2 - // expressions, except in the case where no target expression was - // provided, in which case, the expression is the default value for - // the column; or if the expressions directly map to the source - // table - level1InsertExprs = ((LogicalProject) insertRel.getInput(0)).getProjects(); - if (insertRel.getInput(0).getInput(0) instanceof LogicalProject) { - level2InsertExprs = ((LogicalProject) insertRel.getInput(0).getInput(0)).getProjects(); - } - nLevel1Exprs = level1InsertExprs.size(); - } - - LogicalJoin join = (LogicalJoin) mergeSourceRel.getInput(0); - int nSourceFields = join.getLeft().getRowType().getFieldCount(); - final List projects = new ArrayList<>(); - for (int level1Idx = 0; level1Idx < nLevel1Exprs; level1Idx++) { - if ((level2InsertExprs != null) && (level1InsertExprs.get(level1Idx) instanceof RexInputRef)) { - int level2Idx = ((RexInputRef) level1InsertExprs.get(level1Idx)).getIndex(); - projects.add(level2InsertExprs.get(level2Idx)); - } else { - projects.add(level1InsertExprs.get(level1Idx)); - } - } - if (updateCall != null) { - final LogicalProject project = (LogicalProject) mergeSourceRel; - projects.addAll(Util.skip(project.getProjects(), nSourceFields)); - } - - RelNode massagedRel = RelOptUtil.createProject(join, projects, null, true); - - return LogicalTableModify.create(targetTable, catalogReader, massagedRel, LogicalTableModify.Operation.MERGE, targetColumnNameList, false); - } - - /** - * Converts an identifier into an expression in a given scope. For example, - * the "empno" in "select empno from emp join dept" becomes "emp.empno". - */ - private RexNode convertIdentifier(Blackboard bb, SqlIdentifier identifier) { - // first check for reserved identifiers like CURRENT_USER - final SqlCall call = SqlUtil.makeCall(opTab, identifier); - if (call != null) { - return bb.convertExpression(call); - } - - final SqlQualified qualified; - if (bb.scope != null) { - qualified = bb.scope.fullyQualify(identifier); - identifier = qualified.identifier; - } else { - qualified = SqlQualified.create(null, 1, null, identifier); - } - RexNode e = bb.lookupExp(qualified); - final String correlationName; - if (e instanceof RexCorrelVariable) { - correlationName = ((RexCorrelVariable) e).getName(); - } else { - correlationName = null; - } - - for (String name : qualified.suffixTranslated()) { - final boolean caseSensitive = true; // name already fully-qualified - e = rexBuilder.makeFieldAccess(e, name, caseSensitive); - } - if (e instanceof RexInputRef) { - // adjust the type to account for nulls introduced by outer joins - e = adjustInputRef(bb, (RexInputRef) e); - } - - if (null != correlationName) { - // REVIEW: make mapCorrelateVariableToRexNode map to RexFieldAccess - assert e instanceof RexFieldAccess; - final RexNode prev = bb.mapCorrelateVariableToRexNode.put(correlationName, e); - assert prev == null; - } - return e; - } - - /** - * Adjusts the type of a reference to an input field to account for nulls - * introduced by outer joins; and adjusts the offset to match the physical - * implementation. + *

For example, getHistogramOp + *({@link SqlStdOperatorTable#MIN}} returns + * {@link SqlStdOperatorTable#HISTOGRAM_MIN}. * - * @param bb Blackboard - * @param inputRef Input ref - * @return Adjusted input ref - */ - protected RexNode adjustInputRef(Blackboard bb, RexInputRef inputRef) { - RelDataTypeField field = bb.getRootField(inputRef); - if (field != null) { - return rexBuilder.makeInputRef(field.getType(), inputRef.getIndex()); - } - return inputRef; - } - - /** - * Converts a row constructor into a relational expression. - * - * @param bb Blackboard - * @param rowConstructor Row constructor expression - * @return Relational expression which returns a single row. - * @pre isRowConstructor(rowConstructor) - */ - private RelNode convertRowConstructor(Blackboard bb, SqlCall rowConstructor) { - assert isRowConstructor(rowConstructor) : rowConstructor; - final List operands = rowConstructor.getOperandList(); - return convertMultisets(operands, bb); - } - - private RelNode convertCursor(Blackboard bb, SubQuery subQuery) { - final SqlCall cursorCall = (SqlCall) subQuery.node; - assert cursorCall.operandCount() == 1; - SqlNode query = cursorCall.operand(0); - RelNode converted = convertQuery(query, false, false); - int iCursor = bb.cursors.size(); - bb.cursors.add(converted); - subQuery.expr = new RexInputRef(iCursor, converted.getRowType()); - return converted; - } - - private RelNode convertMultisets(final List operands, Blackboard bb) { - // NOTE: Wael 2/04/05: this implementation is not the most efficient in - // terms of planning since it generates XOs that can be reduced. - final List joinList = new ArrayList<>(); - List lastList = new ArrayList<>(); - for (int i = 0; i < operands.size(); i++) { - SqlNode operand = operands.get(i); - if (!(operand instanceof SqlCall)) { - lastList.add(operand); - continue; - } - - final SqlCall call = (SqlCall) operand; - final SqlOperator op = call.getOperator(); - if ((op != SqlStdOperatorTable.MULTISET_VALUE) && (op != SqlStdOperatorTable.MULTISET_QUERY)) { - lastList.add(operand); - continue; - } - final RelNode input; - if (op == SqlStdOperatorTable.MULTISET_VALUE) { - final SqlNodeList list = new SqlNodeList(call.getOperandList(), call.getParserPosition()); - // assert bb.scope instanceof SelectScope : bb.scope; - CollectNamespace nss = (CollectNamespace) validator.getNamespace(call); - Blackboard usedBb; - if (null != nss) { - usedBb = createBlackboard(nss.getScope(), null); - } else { - usedBb = createBlackboard(new ListScope(bb.scope) { - public SqlNode getNode() { - return call; - } - }, null); - } - RelDataType multisetType = validator.getValidatedNodeType(call); - validator.setValidatedNodeType(list, multisetType.getComponentType()); - input = convertQueryOrInList(usedBb, list); - } else { - input = convertQuery(call.operand(0), false, true); - } - - if (lastList.size() > 0) { - joinList.add(lastList); - } - lastList = new ArrayList<>(); - Collect collect = new Collect(cluster, cluster.traitSetOf(Convention.NONE), input, validator.deriveAlias(call, i)); - joinList.add(collect); - } - - if (joinList.size() == 0) { - joinList.add(lastList); - } - - for (int i = 0; i < joinList.size(); i++) { - Object o = joinList.get(i); - if (o instanceof List) { - @SuppressWarnings("unchecked") - List projectList = (List) o; - final List selectList = new ArrayList<>(); - final List fieldNameList = new ArrayList<>(); - for (int j = 0; j < projectList.size(); j++) { - SqlNode operand = projectList.get(j); - selectList.add(bb.convertExpression(operand)); - - // REVIEW angel 5-June-2005: Use deriveAliasFromOrdinal - // instead of deriveAlias to match field names from - // SqlRowOperator. Otherwise, get error Type - // 'RecordType(INTEGER EMPNO)' has no field 'EXPR$0' when - // doing select * from unnest( select multiset[empno] - // from sales.emps); - - fieldNameList.add(SqlUtil.deriveAliasFromOrdinal(j)); - } - - RelNode projRel = RelOptUtil.createProject(LogicalValues.createOneRow(cluster), selectList, fieldNameList); - - joinList.set(i, projRel); - } - } - - RelNode ret = (RelNode) joinList.get(0); - for (int i = 1; i < joinList.size(); i++) { - RelNode relNode = (RelNode) joinList.get(i); - ret = RelFactories.DEFAULT_JOIN_FACTORY.createJoin(ret, relNode, rexBuilder.makeLiteral(true), JoinRelType.INNER, ImmutableSet. of(), false); - } - return ret; - } - - private void convertSelectList(Blackboard bb, SqlSelect select, List orderList) { - SqlNodeList selectList = select.getSelectList(); - selectList = validator.expandStar(selectList, select, false); - - replaceSubqueries(bb, selectList, RelOptUtil.Logic.TRUE_FALSE_UNKNOWN); - - List fieldNames = new ArrayList<>(); - final List exprs = new ArrayList<>(); - final Collection aliases = new TreeSet<>(); - - // Project any system fields. (Must be done before regular select items, - // because offsets may be affected.) - final List columnMonotonicityList = new ArrayList<>(); - extraSelectItems(bb, select, exprs, fieldNames, aliases, columnMonotonicityList); - - // Project select clause. - int i = -1; - for (SqlNode expr : selectList) { - ++i; - exprs.add(bb.convertExpression(expr)); - fieldNames.add(deriveAlias(expr, aliases, i)); - } - - // Project extra fields for sorting. - for (SqlNode expr : orderList) { - ++i; - SqlNode expr2 = validator.expandOrderExpr(select, expr); - exprs.add(bb.convertExpression(expr2)); - fieldNames.add(deriveAlias(expr, aliases, i)); - } - - fieldNames = SqlValidatorUtil.uniquify(fieldNames); - - RelNode inputRel = bb.root; - bb.setRoot(RelOptUtil.createProject(bb.root, exprs, fieldNames), false); - - assert bb.columnMonotonicities.isEmpty(); - bb.columnMonotonicities.addAll(columnMonotonicityList); - for (SqlNode selectItem : selectList) { - bb.columnMonotonicities.add(selectItem.getMonotonicity(bb.scope)); - } - } - - /** - * Adds extra select items. The default implementation adds nothing; derived - * classes may add columns to exprList, nameList, aliasList and - * columnMonotonicityList. - * - * @param bb Blackboard - * @param select Select statement being translated - * @param exprList List of expressions in select clause - * @param nameList List of names, one per column - * @param aliasList Collection of aliases that have been used - * already - * @param columnMonotonicityList List of monotonicity, one per column - */ - protected void extraSelectItems(Blackboard bb, SqlSelect select, List exprList, List nameList, Collection aliasList, List columnMonotonicityList) { - } - - private String deriveAlias(final SqlNode node, Collection aliases, final int ordinal) { - String alias = validator.deriveAlias(node, ordinal); - if ((alias == null) || aliases.contains(alias)) { - String aliasBase = (alias == null) ? "EXPR$" : alias; - for (int j = 0;; j++) { - alias = aliasBase + j; - if (!aliases.contains(alias)) { - break; - } - } - } - aliases.add(alias); - return alias; - } - - /** - * Converts a WITH sub-query into a relational expression. - */ - public RelNode convertWith(SqlWith with) { - return convertQuery(with.body, false, false); - } - - /** - * Converts a SELECT statement's parse tree into a relational expression. - */ - public RelNode convertValues(SqlCall values, RelDataType targetRowType) { - final SqlValidatorScope scope = validator.getOverScope(values); - assert scope != null; - final Blackboard bb = createBlackboard(scope, null); - convertValuesImpl(bb, values, targetRowType); - return bb.root; - } - - /** - * Converts a values clause (as in "INSERT INTO T(x,y) VALUES (1,2)") into a - * relational expression. - * - * @param bb Blackboard - * @param values Call to SQL VALUES operator - * @param targetRowType Target row type - */ - private void convertValuesImpl(Blackboard bb, SqlCall values, RelDataType targetRowType) { - // Attempt direct conversion to LogicalValues; if that fails, deal with - // fancy stuff like subqueries below. - RelNode valuesRel = convertRowValues(bb, values, values.getOperandList(), true, targetRowType); - if (valuesRel != null) { - bb.setRoot(valuesRel, true); - return; - } - - final List unionRels = new ArrayList<>(); - for (SqlNode rowConstructor1 : values.getOperandList()) { - SqlCall rowConstructor = (SqlCall) rowConstructor1; - Blackboard tmpBb = createBlackboard(bb.scope, null); - replaceSubqueries(tmpBb, rowConstructor, RelOptUtil.Logic.TRUE_FALSE_UNKNOWN); - final List> exps = new ArrayList<>(); - for (Ord operand : Ord.zip(rowConstructor.getOperandList())) { - exps.add(Pair.of(tmpBb.convertExpression(operand.e), validator.deriveAlias(operand.e, operand.i))); - } - RelNode in = (null == tmpBb.root) ? LogicalValues.createOneRow(cluster) : tmpBb.root; - unionRels.add(RelOptUtil.createProject(in, Pair.left(exps), Pair.right(exps), true)); - } - - if (unionRels.size() == 0) { - throw Util.newInternal("empty values clause"); - } else if (unionRels.size() == 1) { - bb.setRoot(unionRels.get(0), true); - } else { - bb.setRoot(LogicalUnion.create(unionRels, true), true); - } - - // REVIEW jvs 22-Jan-2004: should I add - // mapScopeToLux.put(validator.getScope(values),bb.root); - // ? - } - - private String createCorrel() { - int n = nextCorrel++; - return CORREL_PREFIX + n; - } - - private int getCorrelOrdinal(String correlName) { - assert correlName.startsWith(CORREL_PREFIX); - return Integer.parseInt(correlName.substring(CORREL_PREFIX.length())); - } - - //~ Inner Classes ---------------------------------------------------------- - - /** - * Workspace for translating an individual SELECT statement (or sub-SELECT). - */ - protected class Blackboard implements SqlRexContext, SqlVisitor { - /** - * Collection of {@link RelNode} objects which correspond to a SELECT - * statement. - */ - public final SqlValidatorScope scope; - private final Map nameToNodeMap; - public RelNode root; - private List inputs; - private final Map mapCorrelateVariableToRexNode = new HashMap<>(); - - List cursors; - - /** - * List of IN and EXISTS nodes inside this - * SELECT statement (but not inside sub-queries). - */ - private final Set subqueryList = Sets.newLinkedHashSet(); - - private boolean subqueryNeedsOuterJoin; - - /** - * Workspace for building aggregates. - */ - AggConverter agg; - - /** - * When converting window aggregate, we need to know if the window is - * guaranteed to be non-empty. - */ - SqlWindow window; - - /** - * Project the groupby expressions out of the root of this sub-select. - * Subqueries can reference group by expressions projected from the - * "right" to the subquery. - */ - private final Map> mapRootRelToFieldProjection = new HashMap<>(); - - private final List columnMonotonicities = new ArrayList<>(); - - private final List systemFieldList = new ArrayList<>(); - - /** - * Creates a Blackboard. - * - * @param scope Name-resolution scope for expressions validated - * within this query. Can be null if this Blackboard is - * for a leaf node, say - * @param nameToNodeMap Map which translates the expression to map a - * given parameter into, if translating expressions; - * null otherwise - */ - protected Blackboard(SqlValidatorScope scope, Map nameToNodeMap) { - this.scope = scope; - this.nameToNodeMap = nameToNodeMap; - this.cursors = new ArrayList<>(); - subqueryNeedsOuterJoin = false; - } - - public RexNode register(RelNode rel, JoinRelType joinType) { - return register(rel, joinType, null); - } - - /** - * Registers a relational expression. - * - * @param rel Relational expression - * @param joinType Join type - * @param leftKeys LHS of IN clause, or null for expressions - * other than IN - * @return Expression with which to refer to the row (or partial row) - * coming from this relational expression's side of the join - */ - public RexNode register(RelNode rel, JoinRelType joinType, List leftKeys) { - assert joinType != null; - if (root == null) { - assert leftKeys == null; - setRoot(rel, false); - return rexBuilder.makeRangeReference(root.getRowType(), 0, false); - } - - final RexNode joinCond; - final int origLeftInputCount = root.getRowType().getFieldCount(); - if (leftKeys != null) { - List newLeftInputExpr = Lists.newArrayList(); - for (int i = 0; i < origLeftInputCount; i++) { - newLeftInputExpr.add(rexBuilder.makeInputRef(root, i)); - } - - final List leftJoinKeys = Lists.newArrayList(); - for (RexNode leftKey : leftKeys) { - newLeftInputExpr.add(leftKey); - leftJoinKeys.add(origLeftInputCount + leftJoinKeys.size()); - } - - LogicalProject newLeftInput = (LogicalProject) RelOptUtil.createProject(root, newLeftInputExpr, null, true); - - // maintain the group by mapping in the new LogicalProject - if (mapRootRelToFieldProjection.containsKey(root)) { - mapRootRelToFieldProjection.put(newLeftInput, mapRootRelToFieldProjection.get(root)); - } - - setRoot(newLeftInput, false); - - // right fields appear after the LHS fields. - final int rightOffset = root.getRowType().getFieldCount() - newLeftInput.getRowType().getFieldCount(); - final List rightKeys = Util.range(rightOffset, rightOffset + leftJoinKeys.size()); - - joinCond = RelOptUtil.createEquiJoinCondition(newLeftInput, leftJoinKeys, rel, rightKeys, rexBuilder); - } else { - joinCond = rexBuilder.makeLiteral(true); - } - - int leftFieldCount = root.getRowType().getFieldCount(); - final RelNode join = createJoin(this, root, rel, joinCond, joinType); - - setRoot(join, false); - - if (leftKeys != null && joinType == JoinRelType.LEFT) { - final int leftKeyCount = leftKeys.size(); - int rightFieldLength = rel.getRowType().getFieldCount(); - assert leftKeyCount == rightFieldLength - 1; - - final int rexRangeRefLength = leftKeyCount + rightFieldLength; - RelDataType returnType = typeFactory.createStructType(new AbstractList>() { - public Map.Entry get(int index) { - return join.getRowType().getFieldList().get(origLeftInputCount + index); - } - - public int size() { - return rexRangeRefLength; - } - }); - - return rexBuilder.makeRangeReference(returnType, origLeftInputCount, false); - } else { - return rexBuilder.makeRangeReference(rel.getRowType(), leftFieldCount, joinType.generatesNullsOnRight()); - } - } - - /** - * Sets a new root relational expression, as the translation process - * backs its way further up the tree. - * - * @param root New root relational expression - * @param leaf Whether the relational expression is a leaf, that is, - * derived from an atomic relational expression such as a table - * name in the from clause, or the projection on top of a - * select-subquery. In particular, relational expressions - * derived from JOIN operators are not leaves, but set - * expressions are. - */ - public void setRoot(RelNode root, boolean leaf) { - setRoot(Collections.singletonList(root), root, root instanceof LogicalJoin); - if (leaf) { - leaves.add(root); - } - this.columnMonotonicities.clear(); - } - - private void setRoot(List inputs, RelNode root, boolean hasSystemFields) { - this.inputs = inputs; - this.root = root; - this.systemFieldList.clear(); - if (hasSystemFields) { - this.systemFieldList.addAll(getSystemFields()); - } - } - - /** - * Notifies this Blackboard that the root just set using - * {@link #setRoot(RelNode, boolean)} was derived using dataset - * substitution. - * - *

The default implementation is not interested in such - * notifications, and does nothing. - * - * @param datasetName Dataset name - */ - public void setDataset(String datasetName) { - } - - void setRoot(List inputs) { - setRoot(inputs, null, false); - } - - /** - * Returns an expression with which to reference a from-list item. - * - * @param qualified the alias of the from item - * @return a {@link RexFieldAccess} or {@link RexRangeRef}, or null if - * not found - */ - RexNode lookupExp(SqlQualified qualified) { - if (nameToNodeMap != null && qualified.prefixLength == 1) { - RexNode node = nameToNodeMap.get(qualified.identifier.names.get(0)); - if (node == null) { - throw Util.newInternal("Unknown identifier '" + qualified.identifier + "' encountered while expanding expression"); - } - return node; - } - int[] offsets = { -1 }; - final SqlValidatorScope[] ancestorScopes = { null }; - SqlValidatorNamespace foundNs = scope.resolve(qualified.prefix(), ancestorScopes, offsets); - if (foundNs == null) { - return null; - } - - // Found in current query's from list. Find which from item. - // We assume that the order of the from clause items has been - // preserved. - SqlValidatorScope ancestorScope = ancestorScopes[0]; - boolean isParent = ancestorScope != scope; - if ((inputs != null) && !isParent) { - int offset = offsets[0]; - final LookupContext rels = new LookupContext(this, inputs, systemFieldList.size()); - return lookup(offset, rels); - } else { - // We're referencing a relational expression which has not been - // converted yet. This occurs when from items are correlated, - // e.g. "select from emp as emp join emp.getDepts() as dept". - // Create a temporary expression. - assert isParent; - DeferredLookup lookup = new DeferredLookup(this, qualified.identifier.names.get(0)); - String correlName = createCorrel(); - mapCorrelToDeferred.put(correlName, lookup); - final RelDataType rowType = foundNs.getRowType(); - return rexBuilder.makeCorrel(rowType, correlName); - } - } - - /** - * Creates an expression with which to reference the expression whose - * offset in its from-list is {@code offset}. - */ - RexNode lookup(int offset, LookupContext lookupContext) { - Pair pair = lookupContext.findRel(offset); - return rexBuilder.makeRangeReference(pair.left.getRowType(), pair.right, false); - } - - RelDataTypeField getRootField(RexInputRef inputRef) { - int fieldOffset = inputRef.getIndex(); - for (RelNode input : inputs) { - RelDataType rowType = input.getRowType(); - if (rowType == null) { - // TODO: remove this once leastRestrictive - // is correctly implemented - return null; - } - if (fieldOffset < rowType.getFieldCount()) { - return rowType.getFieldList().get(fieldOffset); - } - fieldOffset -= rowType.getFieldCount(); - } - throw new AssertionError(); - } - - public void flatten(List rels, int systemFieldCount, int[] start, List> relOffsetList) { - for (RelNode rel : rels) { - if (leaves.contains(rel)) { - relOffsetList.add(Pair.of(rel, start[0])); - start[0] += rel.getRowType().getFieldCount(); - } else { - if (rel instanceof LogicalJoin || rel instanceof LogicalAggregate) { - start[0] += systemFieldCount; - } - flatten(rel.getInputs(), systemFieldCount, start, relOffsetList); - } - } - } - - void registerSubquery(SqlNode node, RelOptUtil.Logic logic) { - for (SubQuery subQuery : subqueryList) { - if (node.equalsDeep(subQuery.node, false)) { - return; - } - } - subqueryList.add(new SubQuery(node, logic)); - } - - SubQuery getSubquery(SqlNode expr) { - for (SubQuery subQuery : subqueryList) { - if (expr.equalsDeep(subQuery.node, false)) { - return subQuery; - } - } - - return null; - } - - ImmutableList retrieveCursors() { - try { - return ImmutableList.copyOf(cursors); - } finally { - cursors.clear(); - } - } - - public RexNode convertExpression(SqlNode expr) { - // If we're in aggregation mode and this is an expression in the - // GROUP BY clause, return a reference to the field. - if (agg != null) { - final SqlNode expandedGroupExpr = validator.expand(expr, scope); - final int ref = agg.lookupGroupExpr(expandedGroupExpr); - if (ref >= 0) { - return rexBuilder.makeInputRef(root, ref); - } - if (expr instanceof SqlCall) { - final RexNode rex = agg.lookupAggregates((SqlCall) expr); - if (rex != null) { - return rex; - } - } - } - - // Allow the derived class chance to override the standard - // behavior for special kinds of expressions. - RexNode rex = convertExtendedExpression(expr, this); - if (rex != null) { - return rex; - } - - // Sub-queries and OVER expressions are not like ordinary - // expressions. - final SqlKind kind = expr.getKind(); - final SubQuery subQuery; - switch (kind) { - case CURSOR: - case IN: - subQuery = getSubquery(expr); - - assert subQuery != null; - rex = subQuery.expr; - assert rex != null : "rex != null"; - return rex; - - case SELECT: - case EXISTS: - case SCALAR_QUERY: - subQuery = getSubquery(expr); - assert subQuery != null; - rex = subQuery.expr; - assert rex != null : "rex != null"; - - if (((kind == SqlKind.SCALAR_QUERY) || (kind == SqlKind.EXISTS)) && isConvertedSubq(rex)) { - // scalar subquery or EXISTS has been converted to a - // constant - return rex; - } - - // The indicator column is the last field of the subquery. - RexNode fieldAccess = rexBuilder.makeFieldAccess(rex, rex.getType().getFieldCount() - 1); - - // The indicator column will be nullable if it comes from - // the null-generating side of the join. For EXISTS, add an - // "IS TRUE" check so that the result is "BOOLEAN NOT NULL". - if (fieldAccess.getType().isNullable() && kind == SqlKind.EXISTS) { - fieldAccess = rexBuilder.makeCall(SqlStdOperatorTable.IS_NOT_NULL, fieldAccess); - } - return fieldAccess; - - case OVER: - return convertOver(this, expr); - - default: - // fall through - } - - // Apply standard conversions. - rex = expr.accept(this); - Util.permAssert(rex != null, "conversion result not null"); - return rex; - } - - /** - * Converts an item in an ORDER BY clause, extracting DESC, NULLS LAST - * and NULLS FIRST flags first. - */ - public RexNode convertSortExpression(SqlNode expr, Set flags) { - switch (expr.getKind()) { - case DESCENDING: - case NULLS_LAST: - case NULLS_FIRST: - flags.add(expr.getKind()); - final SqlNode operand = ((SqlCall) expr).operand(0); - return convertSortExpression(operand, flags); - default: - return convertExpression(expr); - } - } - - /** - * Determines whether a RexNode corresponds to a subquery that's been - * converted to a constant. - * - * @param rex the expression to be examined - * @return true if the expression is a dynamic parameter, a literal, or - * a literal that is being cast - */ - private boolean isConvertedSubq(RexNode rex) { - if ((rex instanceof RexLiteral) || (rex instanceof RexDynamicParam)) { - return true; - } - if (rex instanceof RexCall) { - RexCall call = (RexCall) rex; - if (call.getOperator() == SqlStdOperatorTable.CAST) { - RexNode operand = call.getOperands().get(0); - if (operand instanceof RexLiteral) { - return true; - } - } - } - return false; - } - - // implement SqlRexContext - public int getGroupCount() { - if (agg != null) { - return agg.groupExprs.size(); - } - if (window != null) { - return window.isAlwaysNonEmpty() ? 1 : 0; - } - return -1; - } - - // implement SqlRexContext - public RexBuilder getRexBuilder() { - return rexBuilder; - } - - // implement SqlRexContext - public RexRangeRef getSubqueryExpr(SqlCall call) { - final SubQuery subQuery = getSubquery(call); - assert subQuery != null; - return (RexRangeRef) subQuery.expr; - } - - // implement SqlRexContext - public RelDataTypeFactory getTypeFactory() { - return typeFactory; - } - - // implement SqlRexContext - public DefaultValueFactory getDefaultValueFactory() { - return defaultValueFactory; - } - - // implement SqlRexContext - public SqlValidator getValidator() { - return validator; - } - - // implement SqlRexContext - public RexNode convertLiteral(SqlLiteral literal) { - return exprConverter.convertLiteral(this, literal); - } - - public RexNode convertInterval(SqlIntervalQualifier intervalQualifier) { - return exprConverter.convertInterval(this, intervalQualifier); - } - - // implement SqlVisitor - public RexNode visit(SqlLiteral literal) { - return exprConverter.convertLiteral(this, literal); - } - - // implement SqlVisitor - public RexNode visit(SqlCall call) { - if (agg != null) { - final SqlOperator op = call.getOperator(); - if (op.isAggregator() || op.getKind() == SqlKind.FILTER) { - return agg.lookupAggregates(call); - } - } - return exprConverter.convertCall(this, call); - } - - // implement SqlVisitor - public RexNode visit(SqlNodeList nodeList) { - throw new UnsupportedOperationException(); - } - - // implement SqlVisitor - public RexNode visit(SqlIdentifier id) { - return convertIdentifier(this, id); - } - - // implement SqlVisitor - public RexNode visit(SqlDataTypeSpec type) { - throw new UnsupportedOperationException(); - } - - // implement SqlVisitor - public RexNode visit(SqlDynamicParam param) { - return convertDynamicParam(param); - } - - // implement SqlVisitor - public RexNode visit(SqlIntervalQualifier intervalQualifier) { - return convertInterval(intervalQualifier); - } - - public List getColumnMonotonicities() { - return columnMonotonicities; - } - } - - /** Deferred lookup. */ - private static class DeferredLookup { - Blackboard bb; - String originalRelName; - - DeferredLookup(Blackboard bb, String originalRelName) { - this.bb = bb; - this.originalRelName = originalRelName; - } - - public RexFieldAccess getFieldAccess(String name) { - return (RexFieldAccess) bb.mapCorrelateVariableToRexNode.get(name); - } - - public String getOriginalRelName() { - return originalRelName; - } - } - - /** - * An implementation of DefaultValueFactory which always supplies NULL. + * @param aggFunction An aggregate function + * @return Its histogram function, or null */ - class NullDefaultValueFactory implements DefaultValueFactory { - public boolean isGeneratedAlways(RelOptTable table, int iColumn) { - return false; - } - - public RexNode newColumnDefaultValue(RelOptTable table, int iColumn) { - return rexBuilder.constantNull(); - } - - public RexNode newAttributeInitializer(RelDataType type, SqlFunction constructor, int iAttribute, List constructorArgs) { - return rexBuilder.constantNull(); - } + SqlFunction getHistogramOp(SqlAggFunction aggFunction) { + if (aggFunction == SqlStdOperatorTable.MIN) { + return SqlStdOperatorTable.HISTOGRAM_MIN; + } else if (aggFunction == SqlStdOperatorTable.MAX) { + return SqlStdOperatorTable.HISTOGRAM_MAX; + } else if (aggFunction == SqlStdOperatorTable.FIRST_VALUE) { + return SqlStdOperatorTable.HISTOGRAM_FIRST_VALUE; + } else if (aggFunction == SqlStdOperatorTable.LAST_VALUE) { + return SqlStdOperatorTable.HISTOGRAM_LAST_VALUE; + } else { + return null; + } } /** - * A default implementation of SubqueryConverter that does no conversion. + * Returns the type for a histogram function. It is either the actual + * type or an an approximation to it. */ - private class NoOpSubqueryConverter implements SubqueryConverter { - // implement SubqueryConverter - public boolean canConvertSubquery() { - return false; - } - - // implement SubqueryConverter - public RexNode convertSubquery(SqlCall subquery, SqlToRelConverter parentConverter, boolean isExists, boolean isExplain) { - throw new IllegalArgumentException(); - } + private RelDataType computeHistogramType(RelDataType type) { + if (SqlTypeUtil.isExactNumeric(type) + && type.getSqlTypeName() != SqlTypeName.BIGINT) { + return typeFactory.createSqlType(SqlTypeName.BIGINT); + } else if (SqlTypeUtil.isApproximateNumeric(type) + && type.getSqlTypeName() != SqlTypeName.DOUBLE) { + return typeFactory.createSqlType(SqlTypeName.DOUBLE); + } else { + return type; + } } - - /** - * Converts expressions to aggregates. - * - *

Consider the expression - * - *

- * {@code SELECT deptno, SUM(2 * sal) FROM emp GROUP BY deptno} - *
- * - *

Then: - * - *

    - *
  • groupExprs = {SqlIdentifier(deptno)}
    • - *
  • convertedInputExprs = {RexInputRef(deptno), 2 * - * RefInputRef(sal)}
    • - *
  • inputRefs = {RefInputRef(#0), RexInputRef(#1)}
    • - *
  • aggCalls = {AggCall(SUM, {1})}
    • - *
- */ - protected class AggConverter implements SqlVisitor { - private final Blackboard bb; - public final AggregatingSelectScope aggregatingSelectScope; - - private final Map nameMap = Maps.newHashMap(); - - /** - * The group-by expressions, in {@link SqlNode} format. - */ - private final SqlNodeList groupExprs = new SqlNodeList(SqlParserPos.ZERO); - - /** - * Input expressions for the group columns and aggregates, in - * {@link RexNode} format. The first elements of the list correspond to the - * elements in {@link #groupExprs}; the remaining elements are for - * aggregates. - */ - private final List convertedInputExprs = Lists.newArrayList(); - - /** - * Names of {@link #convertedInputExprs}, where the expressions are - * simple mappings to input fields. - */ - private final List convertedInputExprNames = Lists.newArrayList(); - - private final List aggCalls = Lists.newArrayList(); - private final Map aggMapping = Maps.newHashMap(); - private final Map aggCallMapping = Maps.newHashMap(); - - /** - * Creates an AggConverter. - * - *

The select parameter provides enough context to name - * aggregate calls which are top-level select list items. - * - * @param bb Blackboard - * @param select Query being translated; provides context to give - */ - public AggConverter(Blackboard bb, SqlSelect select) { - this.bb = bb; - this.aggregatingSelectScope = (AggregatingSelectScope) bb.getValidator().getSelectScope(select); - - // Collect all expressions used in the select list so that aggregate - // calls can be named correctly. - final SqlNodeList selectList = select.getSelectList(); - for (int i = 0; i < selectList.size(); i++) { - SqlNode selectItem = selectList.get(i); - String name = null; - if (SqlUtil.isCallTo(selectItem, SqlStdOperatorTable.AS)) { - final SqlCall call = (SqlCall) selectItem; - selectItem = call.operand(0); - name = call.operand(1).toString(); - } - if (name == null) { - name = validator.deriveAlias(selectItem, i); - } - nameMap.put(selectItem.toString(), name); - } - } - - public int addGroupExpr(SqlNode expr) { - RexNode convExpr = bb.convertExpression(expr); - int ref = lookupGroupExpr(expr); - if (ref >= 0) { - return ref; - } - final int index = groupExprs.size(); - groupExprs.add(expr); - String name = nameMap.get(expr.toString()); - addExpr(convExpr, name); - return index; - } - - /** - * Adds an expression, deducing an appropriate name if possible. - * - * @param expr Expression - * @param name Suggested name - */ - private void addExpr(RexNode expr, String name) { - convertedInputExprs.add(expr); - if ((name == null) && (expr instanceof RexInputRef)) { - final int i = ((RexInputRef) expr).getIndex(); - name = bb.root.getRowType().getFieldList().get(i).getName(); - } - if (convertedInputExprNames.contains(name)) { - // In case like 'SELECT ... GROUP BY x, y, x', don't add - // name 'x' twice. - name = null; - } - convertedInputExprNames.add(name); - } - - // implement SqlVisitor - public Void visit(SqlIdentifier id) { - return null; - } - - // implement SqlVisitor - public Void visit(SqlNodeList nodeList) { - for (int i = 0; i < nodeList.size(); i++) { - nodeList.get(i).accept(this); - } - return null; - } - - // implement SqlVisitor - public Void visit(SqlLiteral lit) { - return null; - } - - // implement SqlVisitor - public Void visit(SqlDataTypeSpec type) { - return null; - } - - // implement SqlVisitor - public Void visit(SqlDynamicParam param) { - return null; - } - - // implement SqlVisitor - public Void visit(SqlIntervalQualifier intervalQualifier) { - return null; - } - - public Void visit(SqlCall call) { - switch (call.getKind()) { - case FILTER: - translateAgg((SqlCall) call.operand(0), call.operand(1), call); - return null; - case SELECT: - // rchen 2006-10-17: - // for now do not detect aggregates in subqueries. - return null; - } - // ignore window aggregates and ranking functions (associated with OVER operator) - if (call.getOperator().getKind() == SqlKind.OVER) { - return null; - } - if (call.getOperator().isAggregator()) { - translateAgg(call, null, call); - return null; - } - for (SqlNode operand : call.getOperandList()) { - // Operands are occasionally null, e.g. switched CASE arg 0. - if (operand != null) { - operand.accept(this); - } - } - return null; - } - - private void translateAgg(SqlCall call, SqlNode filter, SqlCall outerCall) { - assert bb.agg == this; - final List args = new ArrayList<>(); - int filterArg = -1; - final List argTypes = call.getOperator() instanceof SqlCountAggFunction ? new ArrayList(call.getOperandList().size()) : null; - try { - // switch out of agg mode - bb.agg = null; - for (SqlNode operand : call.getOperandList()) { - - // special case for COUNT(*): delete the * - if (operand instanceof SqlIdentifier) { - SqlIdentifier id = (SqlIdentifier) operand; - if (id.isStar() || isSimpleCount(call)) { /* OVERRIDE POINT */ - assert call.operandCount() == 1; - assert args.isEmpty(); - break; - } - } - RexNode convertedExpr = bb.convertExpression(operand); - assert convertedExpr != null; - if (argTypes != null) { - argTypes.add(convertedExpr.getType()); - } - args.add(lookupOrCreateGroupExpr(convertedExpr)); - } - - if (filter != null) { - RexNode convertedExpr = bb.convertExpression(filter); - assert convertedExpr != null; - filterArg = lookupOrCreateGroupExpr(convertedExpr); - } - } finally { - // switch back into agg mode - bb.agg = this; - } - - final SqlAggFunction aggFunction = (SqlAggFunction) call.getOperator(); - RelDataType type = validator.deriveType(bb.scope, call); - boolean distinct = false; - SqlLiteral quantifier = call.getFunctionQuantifier(); - if ((null != quantifier) && (quantifier.getValue() == SqlSelectKeyword.DISTINCT)) { - distinct = true; - } - final AggregateCall aggCall = AggregateCall.create(aggFunction, distinct, args, filterArg, type, nameMap.get(outerCall.toString())); - RexNode rex = rexBuilder.addAggCall(aggCall, groupExprs.size(), aggregatingSelectScope.indicator, aggCalls, aggCallMapping, argTypes); - aggMapping.put(outerCall, rex); - } - - /* OVERRIDE POINT */ - private boolean isSimpleCount(SqlCall call) { - if (call.getOperator().isName("COUNT") && call.operandCount() == 1) { - final SqlNode parm = call.operand(0); - if ((parm instanceof SqlIdentifier || parm instanceof SqlNumericLiteral) // - && call.getFunctionQuantifier() == null) { - return true; - } - } - return false; - } - - private int lookupOrCreateGroupExpr(RexNode expr) { - for (int i = 0; i < convertedInputExprs.size(); i++) { - RexNode convertedInputExpr = convertedInputExprs.get(i); - if (expr.toString().equals(convertedInputExpr.toString())) { - return i; - } - } - - // not found -- add it - int index = convertedInputExprs.size(); - addExpr(expr, null); - return index; - } - - /** - * If an expression is structurally identical to one of the group-by - * expressions, returns a reference to the expression, otherwise returns - * null. - */ - public int lookupGroupExpr(SqlNode expr) { - for (int i = 0; i < groupExprs.size(); i++) { - SqlNode groupExpr = groupExprs.get(i); - if (expr.equalsDeep(groupExpr, false)) { - return i; - } - } - return -1; - } - - public RexNode lookupAggregates(SqlCall call) { - // assert call.getOperator().isAggregator(); - assert bb.agg == this; - - switch (call.getKind()) { - case GROUPING: - case GROUPING_ID: - case GROUP_ID: - final RelDataType type = validator.getValidatedNodeType(call); - if (!aggregatingSelectScope.indicator) { - return rexBuilder.makeExactLiteral(TWO.pow(effectiveArgCount(call)).subtract(BigDecimal.ONE), type); - } else { - final List operands; - switch (call.getKind()) { - case GROUP_ID: - operands = ImmutableIntList.range(0, groupExprs.size()); - break; - default: - operands = Lists.newArrayList(); - for (SqlNode operand : call.getOperandList()) { - final int x = lookupGroupExpr(operand); - assert x >= 0; - operands.add(x); - } - } - RexNode node = null; - int shift = operands.size(); - for (int operand : operands) { - node = bitValue(node, type, operand, --shift); - } - return node; - } - } - return aggMapping.get(call); - } - - private int effectiveArgCount(SqlCall call) { - switch (call.getKind()) { - case GROUPING: - return 1; - case GROUPING_ID: - return call.operandCount(); - case GROUP_ID: - return groupExprs.size(); - default: - throw new AssertionError(call.getKind()); - } - } - - private RexNode bitValue(RexNode previous, RelDataType type, int x, int shift) { - RexNode node = rexBuilder.makeCall(SqlStdOperatorTable.CASE, rexBuilder.makeInputRef(bb.root, aggregatingSelectScope.groupExprList.size() + x), rexBuilder.makeExactLiteral(BigDecimal.ONE, type), rexBuilder.makeExactLiteral(BigDecimal.ZERO, type)); - if (shift > 0) { - node = rexBuilder.makeCall(SqlStdOperatorTable.MULTIPLY, node, rexBuilder.makeExactLiteral(TWO.pow(shift), type)); - } - if (previous != null) { - node = rexBuilder.makeCall(SqlStdOperatorTable.PLUS, previous, node); - } - return node; - } - - public List getPreExprs() { - return convertedInputExprs; - } - - public List getPreNames() { - return convertedInputExprNames; - } - - public List getAggCalls() { - return aggCalls; - } - - public RelDataTypeFactory getTypeFactory() { - return typeFactory; - } + } + + /** A sub-query, whether it needs to be translated using 2- or 3-valued + * logic. */ + private static class SubQuery { + final SqlNode node; + final RelOptUtil.Logic logic; + RexNode expr; + + private SubQuery(SqlNode node, RelOptUtil.Logic logic) { + this.node = node; + this.logic = logic; } + } - /** - * Context to find a relational expression to a field offset. - */ - private static class LookupContext { - private final List> relOffsetList = new ArrayList<>(); - - /** - * Creates a LookupContext with multiple input relational expressions. - * - * @param bb Context for translating this subquery - * @param rels Relational expressions - * @param systemFieldCount Number of system fields - */ - LookupContext(Blackboard bb, List rels, int systemFieldCount) { - bb.flatten(rels, systemFieldCount, new int[] { 0 }, relOffsetList); - } - - /** - * Returns the relational expression with a given offset, and the - * ordinal in the combined row of its first field. - * - *

For example, in {@code Emp JOIN Dept}, findRel(1) returns the - * relational expression for {@code Dept} and offset 6 (because - * {@code Emp} has 6 fields, therefore the first field of {@code Dept} - * is field 6. - * - * @param offset Offset of relational expression in FROM clause - * @return Relational expression and the ordinal of its first field - */ - Pair findRel(int offset) { - return relOffsetList.get(offset); - } - } - - /** - * Shuttle which walks over a tree of {@link RexNode}s and applies 'over' to - * all agg functions. - * - *

This is necessary because the returned expression is not necessarily a - * call to an agg function. For example, - * - *

AVG(x)
- * - * becomes - * - *
SUM(x) / COUNT(x)
- * - *

Any aggregate functions are converted to calls to the internal - * $Histogram aggregation function and accessors such as - * $HistogramMin; for example, - * - *

MIN(x), MAX(x)
- * - * are converted to - * - *
$HistogramMin($Histogram(x)), - * $HistogramMax($Histogram(x))
- * - * Common sub-expression elmination will ensure that only one histogram is - * computed. - */ - private class HistogramShuttle extends RexShuttle { - /** - * Whether to convert calls to MIN(x) to HISTOGRAM_MIN(HISTOGRAM(x)). - * Histograms allow rolling computation, but require more space. - */ - static final boolean ENABLE_HISTOGRAM_AGG = false; - - private final List partitionKeys; - private final ImmutableList orderKeys; - private final RexWindowBound lowerBound; - private final RexWindowBound upperBound; - private final SqlWindow window; - - HistogramShuttle(List partitionKeys, ImmutableList orderKeys, RexWindowBound lowerBound, RexWindowBound upperBound, SqlWindow window) { - this.partitionKeys = partitionKeys; - this.orderKeys = orderKeys; - this.lowerBound = lowerBound; - this.upperBound = upperBound; - this.window = window; - } - - public RexNode visitCall(RexCall call) { - final SqlOperator op = call.getOperator(); - if (!(op instanceof SqlAggFunction)) { - return super.visitCall(call); - } - final SqlAggFunction aggOp = (SqlAggFunction) op; - final RelDataType type = call.getType(); - List exprs = call.getOperands(); - - SqlFunction histogramOp = !ENABLE_HISTOGRAM_AGG ? null : getHistogramOp(aggOp); - - if (histogramOp != null) { - final RelDataType histogramType = computeHistogramType(type); - - // For DECIMAL, since it's already represented as a bigint we - // want to do a reinterpretCast instead of a cast to avoid - // losing any precision. - boolean reinterpretCast = type.getSqlTypeName() == SqlTypeName.DECIMAL; + /** + * Visitor that collects all aggregate functions in a {@link SqlNode} tree. + */ + private static class AggregateFinder extends SqlBasicVisitor { + final SqlNodeList list = new SqlNodeList(SqlParserPos.ZERO); - // Replace original expression with CAST of not one - // of the supported types - if (histogramType != type) { - exprs = new ArrayList<>(exprs); - exprs.set(0, reinterpretCast ? rexBuilder.makeReinterpretCast(histogramType, exprs.get(0), rexBuilder.makeLiteral(false)) : rexBuilder.makeCast(histogramType, exprs.get(0))); - } - - RexCallBinding bind = new RexCallBinding(rexBuilder.getTypeFactory(), SqlStdOperatorTable.HISTOGRAM_AGG, exprs, ImmutableList. of()); - - RexNode over = rexBuilder.makeOver(SqlStdOperatorTable.HISTOGRAM_AGG.inferReturnType(bind), SqlStdOperatorTable.HISTOGRAM_AGG, exprs, partitionKeys, orderKeys, lowerBound, upperBound, window.isRows(), window.isAllowPartial(), false); - - RexNode histogramCall = rexBuilder.makeCall(histogramType, histogramOp, ImmutableList.of(over)); - - // If needed, post Cast result back to original - // type. - if (histogramType != type) { - if (reinterpretCast) { - histogramCall = rexBuilder.makeReinterpretCast(type, histogramCall, rexBuilder.makeLiteral(false)); - } else { - histogramCall = rexBuilder.makeCast(type, histogramCall); - } - } - - return histogramCall; - } else { - boolean needSum0 = aggOp == SqlStdOperatorTable.SUM && type.isNullable(); - SqlAggFunction aggOpToUse = needSum0 ? SqlStdOperatorTable.SUM0 : aggOp; - return rexBuilder.makeOver(type, aggOpToUse, exprs, partitionKeys, orderKeys, lowerBound, upperBound, window.isRows(), window.isAllowPartial(), needSum0); - } - } - - /** - * Returns the histogram operator corresponding to a given aggregate - * function. - * - *

For example, getHistogramOp - *({@link SqlStdOperatorTable#MIN}} returns - * {@link SqlStdOperatorTable#HISTOGRAM_MIN}. - * - * @param aggFunction An aggregate function - * @return Its histogram function, or null - */ - SqlFunction getHistogramOp(SqlAggFunction aggFunction) { - if (aggFunction == SqlStdOperatorTable.MIN) { - return SqlStdOperatorTable.HISTOGRAM_MIN; - } else if (aggFunction == SqlStdOperatorTable.MAX) { - return SqlStdOperatorTable.HISTOGRAM_MAX; - } else if (aggFunction == SqlStdOperatorTable.FIRST_VALUE) { - return SqlStdOperatorTable.HISTOGRAM_FIRST_VALUE; - } else if (aggFunction == SqlStdOperatorTable.LAST_VALUE) { - return SqlStdOperatorTable.HISTOGRAM_LAST_VALUE; - } else { - return null; - } - } - - /** - * Returns the type for a histogram function. It is either the actual - * type or an an approximation to it. - */ - private RelDataType computeHistogramType(RelDataType type) { - if (SqlTypeUtil.isExactNumeric(type) && type.getSqlTypeName() != SqlTypeName.BIGINT) { - return typeFactory.createSqlType(SqlTypeName.BIGINT); - } else if (SqlTypeUtil.isApproximateNumeric(type) && type.getSqlTypeName() != SqlTypeName.DOUBLE) { - return typeFactory.createSqlType(SqlTypeName.DOUBLE); - } else { - return type; - } - } - } + @Override public Void visit(SqlCall call) { + // ignore window aggregates and ranking functions (associated with OVER operator) + if (call.getOperator().getKind() == SqlKind.OVER) { + return null; + } + if (call.getOperator().isAggregator()) { + list.add(call); + return null; + } - /** A sub-query, whether it needs to be translated using 2- or 3-valued - * logic. */ - private static class SubQuery { - final SqlNode node; - final RelOptUtil.Logic logic; - RexNode expr; + // Don't traverse into sub-queries, even if they contain aggregate + // functions. + if (call instanceof SqlSelect) { + return null; + } - private SubQuery(SqlNode node, RelOptUtil.Logic logic) { - this.node = node; - this.logic = logic; - } + return call.getOperator().acceptCall(this, call); } - - /** - * Visitor that collects all aggregate functions in a {@link SqlNode} tree. - */ - private static class AggregateFinder extends SqlBasicVisitor { - final SqlNodeList list = new SqlNodeList(SqlParserPos.ZERO); - - @Override - public Void visit(SqlCall call) { - // ignore window aggregates and ranking functions (associated with OVER operator) - if (call.getOperator().getKind() == SqlKind.OVER) { - return null; - } - if (call.getOperator().isAggregator()) { - list.add(call); - return null; - } - - // Don't traverse into sub-queries, even if they contain aggregate - // functions. - if (call instanceof SqlSelect) { - return null; - } - - return call.getOperator().acceptCall(this, call); - } + } + + /** Use of a row as a correlating variable by a given relational + * expression. */ + private static class CorrelationUse { + private final CorrelationId id; + private final ImmutableBitSet requiredColumns; + private final RelNode r; + + CorrelationUse(CorrelationId id, ImmutableBitSet requiredColumns, + RelNode r) { + this.id = id; + this.requiredColumns = requiredColumns; + this.r = r; } + } } // End SqlToRelConverter.java diff --git a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MergeCuboidJobTest.java b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MergeCuboidJobTest.java index 7e530db..97b1ef2 100644 --- a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MergeCuboidJobTest.java +++ b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MergeCuboidJobTest.java @@ -44,7 +44,6 @@ public class MergeCuboidJobTest extends LocalFileMetadataTestCase { conf.set("fs.default.name", "file:///"); conf.set("mapreduce.framework.name", "local"); conf.set("mapreduce.application.framework.path", ""); - // for local runner out-of-memory issue conf.set("mapreduce.task.io.sort.mb", "10"); diff --git a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinConnection.java b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinConnection.java index 86c8d1d..6852998 100644 --- a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinConnection.java +++ b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinConnection.java @@ -118,7 +118,7 @@ public class KylinConnection extends AvaticaConnection { ArrayList columns = new ArrayList(); Map internalParams = Collections. emptyMap(); - return new Meta.Signature(columns, sql, params, internalParams, CursorFactory.ARRAY); + return new Meta.Signature(columns, sql, params, internalParams, CursorFactory.ARRAY, Meta.StatementType.SELECT); } private KylinJdbcFactory factory() { diff --git a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinJdbcFactory.java b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinJdbcFactory.java index f1a4939..6aae983 100644 --- a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinJdbcFactory.java +++ b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinJdbcFactory.java @@ -33,6 +33,7 @@ import org.apache.calcite.avatica.AvaticaStatement; import org.apache.calcite.avatica.Meta.Frame; import org.apache.calcite.avatica.Meta.Signature; import org.apache.calcite.avatica.Meta.StatementHandle; +import org.apache.calcite.avatica.QueryState; import org.apache.calcite.avatica.UnregisteredDriver; /** @@ -93,9 +94,9 @@ public class KylinJdbcFactory implements AvaticaFactory { } @Override - public AvaticaResultSet newResultSet(AvaticaStatement statement, Signature signature, TimeZone timeZone, Frame firstFrame) throws SQLException { + public AvaticaResultSet newResultSet(AvaticaStatement statement, QueryState state, Signature signature, TimeZone timeZone, Frame firstFrame) throws SQLException { AvaticaResultSetMetaData resultSetMetaData = new AvaticaResultSetMetaData(statement, null, signature); - return new KylinResultSet(statement, signature, resultSetMetaData, timeZone, firstFrame); + return new KylinResultSet(statement, state, signature, resultSetMetaData, timeZone, firstFrame); } @Override diff --git a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinMeta.java b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinMeta.java index cae39ad..05f7983 100644 --- a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinMeta.java +++ b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinMeta.java @@ -30,6 +30,10 @@ import java.util.regex.Pattern; import org.apache.calcite.avatica.AvaticaUtils; import org.apache.calcite.avatica.ColumnMetaData; import org.apache.calcite.avatica.MetaImpl; +import org.apache.calcite.avatica.MissingResultsException; +import org.apache.calcite.avatica.NoSuchStatementException; +import org.apache.calcite.avatica.QueryState; +import org.apache.calcite.avatica.remote.TypedValue; import com.google.common.collect.ImmutableList; @@ -55,6 +59,13 @@ public class KylinMeta extends MetaImpl { result.signature = connection().mockPreparedSignature(sql); return result; } + + // real execution happens in KylinResultSet.execute() + @Override + public ExecuteResult execute(StatementHandle sh, List parameterValues, long maxRowCount) throws NoSuchStatementException { + final MetaResultSet metaResultSet = MetaResultSet.create(sh.connectionId, sh.id, false, sh.signature, null); + return new ExecuteResult(ImmutableList.of(metaResultSet)); + } // mimic from CalciteMetaImpl, real execution happens via callback in KylinResultSet.execute() @Override @@ -91,24 +102,24 @@ public class KylinMeta extends MetaImpl { } @Override - public MetaResultSet getTableTypes() { + public MetaResultSet getTableTypes(ConnectionHandle ch) { return createResultSet(metaTableTypes, MetaTableType.class, "TABLE_TYPE"); } @Override - public MetaResultSet getCatalogs() { + public MetaResultSet getCatalogs(ConnectionHandle ch) { List catalogs = getMetaProject().catalogs; return createResultSet(catalogs, KMetaCatalog.class, "TABLE_CAT"); } @Override - public MetaResultSet getSchemas(String catalog, Pat schemaPattern) { + public MetaResultSet getSchemas(ConnectionHandle ch, String catalog, Pat schemaPattern) { List schemas = getMetaProject().getSchemas(catalog, schemaPattern); return createResultSet(schemas, KMetaSchema.class, "TABLE_SCHEM", "TABLE_CATALOG"); } @Override - public MetaResultSet getTables(String catalog, Pat schemaPattern, Pat tableNamePattern, List typeList) { + public MetaResultSet getTables(ConnectionHandle ch, String catalog, Pat schemaPattern, Pat tableNamePattern, List typeList) { List tables = getMetaProject().getTables(catalog, schemaPattern, tableNamePattern, typeList); return createResultSet(tables, KMetaTable.class, // "TABLE_CAT", // @@ -124,7 +135,7 @@ public class KylinMeta extends MetaImpl { } @Override - public MetaResultSet getColumns(String catalog, Pat schemaPattern, Pat tableNamePattern, Pat columnNamePattern) { + public MetaResultSet getColumns(ConnectionHandle ch, String catalog, Pat schemaPattern, Pat tableNamePattern, Pat columnNamePattern) { List columns = getMetaProject().getColumns(catalog, schemaPattern, tableNamePattern, columnNamePattern); return createResultSet(columns, KMetaColumn.class, // "TABLE_CAT", // @@ -172,7 +183,7 @@ public class KylinMeta extends MetaImpl { } CursorFactory cursorFactory = CursorFactory.record(clazz, fields, fieldNames); - Signature signature = new Signature(columns, "", null, Collections. emptyMap(), cursorFactory); + Signature signature = new Signature(columns, "", null, Collections. emptyMap(), cursorFactory, StatementType.SELECT); StatementHandle sh = this.createStatement(connection().handle); Frame frame = new Frame(0, true, iterable); @@ -355,4 +366,27 @@ public class KylinMeta extends MetaImpl { } } + @Override + public Frame fetch(StatementHandle h, long offset, int fetchMaxRowCount) throws NoSuchStatementException, MissingResultsException { + // TODO Auto-generated method stub + return null; + } + + @Override + public boolean syncResults(StatementHandle sh, QueryState state, long offset) throws NoSuchStatementException { + // TODO Auto-generated method stub + return false; + } + + @Override + public void commit(ConnectionHandle ch) { + // TODO Auto-generated method stub + + } + + @Override + public void rollback(ConnectionHandle ch) { + // TODO Auto-generated method stub + + } } diff --git a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinPreparedStatement.java b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinPreparedStatement.java index b14865b..04567cb 100644 --- a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinPreparedStatement.java +++ b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinPreparedStatement.java @@ -18,6 +18,8 @@ public class KylinPreparedStatement extends AvaticaPreparedStatement { protected KylinPreparedStatement(AvaticaConnection connection, StatementHandle h, Signature signature, int resultSetType, int resultSetConcurrency, int resultSetHoldability) throws SQLException { super(connection, h, signature, resultSetType, resultSetConcurrency, resultSetHoldability); + if (this.handle.signature == null) + this.handle.signature = signature; } protected List getParameterValues2() { diff --git a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinResultSet.java b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinResultSet.java index b6ac261..2b186bf 100644 --- a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinResultSet.java +++ b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinResultSet.java @@ -30,12 +30,13 @@ import org.apache.calcite.avatica.AvaticaStatement; import org.apache.calcite.avatica.Meta.Frame; import org.apache.calcite.avatica.Meta.Signature; import org.apache.calcite.avatica.MetaImpl; +import org.apache.calcite.avatica.QueryState; import org.apache.kylin.jdbc.IRemoteClient.QueryResult; public class KylinResultSet extends AvaticaResultSet { - public KylinResultSet(AvaticaStatement statement, Signature signature, ResultSetMetaData resultSetMetaData, TimeZone timeZone, Frame firstFrame) { - super(statement, signature, resultSetMetaData, timeZone, firstFrame); + public KylinResultSet(AvaticaStatement statement, QueryState state, Signature signature, ResultSetMetaData resultSetMetaData, TimeZone timeZone, Frame firstFrame) { + super(statement, state, signature, resultSetMetaData, timeZone, firstFrame); } @Override diff --git a/kylin-it/src/test/resources/logging.properties b/kylin-it/src/test/resources/logging.properties index a925478..3b201a1 100644 --- a/kylin-it/src/test/resources/logging.properties +++ b/kylin-it/src/test/resources/logging.properties @@ -1,5 +1,5 @@ handlers=java.util.logging.ConsoleHandler .level=INFO -#org.eigenbase.relopt.RelOptPlanner.level=FINEST +#org.apache.calcite.plan.RelOptPlanner.level=FINE java.util.logging.ConsoleHandler.level=ALL java.util.logging.ConsoleHandler.formatter=org.apache.kylin.common.util.MyLogFormatter \ No newline at end of file diff --git a/kylin-it/src/test/resources/query/sql/query92.sql b/kylin-it/src/test/resources/query/sql/query92.sql new file mode 100644 index 0000000..e551a45 --- /dev/null +++ b/kylin-it/src/test/resources/query/sql/query92.sql @@ -0,0 +1,30 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- + +select meta_categ_name, count(1) as cnt, sum(price) as GMV + + from test_kylin_fact + left JOIN edw.test_cal_dt as test_cal_dt + ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt + left JOIN test_category_groupings + ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND test_kylin_fact.lstg_site_id = test_category_groupings.site_id + left JOIN edw.test_sites as test_sites + ON test_kylin_fact.lstg_site_id = test_sites.site_id + + where meta_categ_name not in ('', 'a') + group by meta_categ_name diff --git a/kylin-it/src/test/resources/query/sql/query92.sql.disabled b/kylin-it/src/test/resources/query/sql/query92.sql.disabled deleted file mode 100644 index e551a45..0000000 --- a/kylin-it/src/test/resources/query/sql/query92.sql.disabled +++ /dev/null @@ -1,30 +0,0 @@ --- --- Licensed to the Apache Software Foundation (ASF) under one --- or more contributor license agreements. See the NOTICE file --- distributed with this work for additional information --- regarding copyright ownership. The ASF licenses this file --- to you under the Apache License, Version 2.0 (the --- "License"); you may not use this file except in compliance --- with the License. You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- - -select meta_categ_name, count(1) as cnt, sum(price) as GMV - - from test_kylin_fact - left JOIN edw.test_cal_dt as test_cal_dt - ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt - left JOIN test_category_groupings - ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND test_kylin_fact.lstg_site_id = test_category_groupings.site_id - left JOIN edw.test_sites as test_sites - ON test_kylin_fact.lstg_site_id = test_sites.site_id - - where meta_categ_name not in ('', 'a') - group by meta_categ_name diff --git a/kylin-it/src/test/resources/query/sql/query93.sql b/kylin-it/src/test/resources/query/sql/query93.sql new file mode 100644 index 0000000..cc6dca5 --- /dev/null +++ b/kylin-it/src/test/resources/query/sql/query93.sql @@ -0,0 +1,30 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- + +select meta_categ_name, count(1) as cnt, sum(price) as GMV + + from test_kylin_fact + left JOIN edw.test_cal_dt as test_cal_dt + ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt + left JOIN test_category_groupings + ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND test_kylin_fact.lstg_site_id = test_category_groupings.site_id + left JOIN edw.test_sites as test_sites + ON test_kylin_fact.lstg_site_id = test_sites.site_id + + where meta_categ_name is not null + group by meta_categ_name diff --git a/kylin-it/src/test/resources/query/sql/query93.sql.disabled b/kylin-it/src/test/resources/query/sql/query93.sql.disabled deleted file mode 100644 index cc6dca5..0000000 --- a/kylin-it/src/test/resources/query/sql/query93.sql.disabled +++ /dev/null @@ -1,30 +0,0 @@ --- --- Licensed to the Apache Software Foundation (ASF) under one --- or more contributor license agreements. See the NOTICE file --- distributed with this work for additional information --- regarding copyright ownership. The ASF licenses this file --- to you under the Apache License, Version 2.0 (the --- "License"); you may not use this file except in compliance --- with the License. You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- - -select meta_categ_name, count(1) as cnt, sum(price) as GMV - - from test_kylin_fact - left JOIN edw.test_cal_dt as test_cal_dt - ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt - left JOIN test_category_groupings - ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND test_kylin_fact.lstg_site_id = test_category_groupings.site_id - left JOIN edw.test_sites as test_sites - ON test_kylin_fact.lstg_site_id = test_sites.site_id - - where meta_categ_name is not null - group by meta_categ_name diff --git a/kylin-it/src/test/resources/query/sql/query94.sql b/kylin-it/src/test/resources/query/sql/query94.sql new file mode 100644 index 0000000..c7899fd --- /dev/null +++ b/kylin-it/src/test/resources/query/sql/query94.sql @@ -0,0 +1,30 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- + +select meta_categ_name, count(1) as cnt, sum(price) as GMV + + from test_kylin_fact + left JOIN edw.test_cal_dt as test_cal_dt + ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt + left JOIN test_category_groupings + ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND test_kylin_fact.lstg_site_id = test_category_groupings.site_id + left JOIN edw.test_sites as test_sites + ON test_kylin_fact.lstg_site_id = test_sites.site_id + + where meta_categ_name not in ('Unknown') + group by meta_categ_name diff --git a/kylin-it/src/test/resources/query/sql/query94.sql.disabled b/kylin-it/src/test/resources/query/sql/query94.sql.disabled deleted file mode 100644 index c7899fd..0000000 --- a/kylin-it/src/test/resources/query/sql/query94.sql.disabled +++ /dev/null @@ -1,30 +0,0 @@ --- --- Licensed to the Apache Software Foundation (ASF) under one --- or more contributor license agreements. See the NOTICE file --- distributed with this work for additional information --- regarding copyright ownership. The ASF licenses this file --- to you under the Apache License, Version 2.0 (the --- "License"); you may not use this file except in compliance --- with the License. You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- - -select meta_categ_name, count(1) as cnt, sum(price) as GMV - - from test_kylin_fact - left JOIN edw.test_cal_dt as test_cal_dt - ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt - left JOIN test_category_groupings - ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND test_kylin_fact.lstg_site_id = test_category_groupings.site_id - left JOIN edw.test_sites as test_sites - ON test_kylin_fact.lstg_site_id = test_sites.site_id - - where meta_categ_name not in ('Unknown') - group by meta_categ_name diff --git a/kylin-it/src/test/resources/query/sql/query95.sql b/kylin-it/src/test/resources/query/sql/query95.sql new file mode 100644 index 0000000..578b93f --- /dev/null +++ b/kylin-it/src/test/resources/query/sql/query95.sql @@ -0,0 +1,30 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- + +select meta_categ_name, count(1) as cnt, sum(price) as GMV + + from test_kylin_fact + left JOIN edw.test_cal_dt as test_cal_dt + ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt + left JOIN test_category_groupings + ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND test_kylin_fact.lstg_site_id = test_category_groupings.site_id + left JOIN edw.test_sites as test_sites + ON test_kylin_fact.lstg_site_id = test_sites.site_id + + where meta_categ_name not in ('Unknown', 'ToyHobbies', '', 'a', 'BookMagazines') + group by meta_categ_name diff --git a/kylin-it/src/test/resources/query/sql/query95.sql.disabled b/kylin-it/src/test/resources/query/sql/query95.sql.disabled deleted file mode 100644 index 578b93f..0000000 --- a/kylin-it/src/test/resources/query/sql/query95.sql.disabled +++ /dev/null @@ -1,30 +0,0 @@ --- --- Licensed to the Apache Software Foundation (ASF) under one --- or more contributor license agreements. See the NOTICE file --- distributed with this work for additional information --- regarding copyright ownership. The ASF licenses this file --- to you under the Apache License, Version 2.0 (the --- "License"); you may not use this file except in compliance --- with the License. You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- - -select meta_categ_name, count(1) as cnt, sum(price) as GMV - - from test_kylin_fact - left JOIN edw.test_cal_dt as test_cal_dt - ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt - left JOIN test_category_groupings - ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND test_kylin_fact.lstg_site_id = test_category_groupings.site_id - left JOIN edw.test_sites as test_sites - ON test_kylin_fact.lstg_site_id = test_sites.site_id - - where meta_categ_name not in ('Unknown', 'ToyHobbies', '', 'a', 'BookMagazines') - group by meta_categ_name diff --git a/pom.xml b/pom.xml index 2e42841..501bf77 100644 --- a/pom.xml +++ b/pom.xml @@ -99,7 +99,7 @@ 1.2.7.RELEASE - 1.4.0-incubating + 1.6.0 2.6.0 diff --git a/query/src/main/java/org/apache/kylin/query/optrule/OLAPJoinRule.java b/query/src/main/java/org/apache/kylin/query/optrule/OLAPJoinRule.java index 10047e6..c31d1d0 100644 --- a/query/src/main/java/org/apache/kylin/query/optrule/OLAPJoinRule.java +++ b/query/src/main/java/org/apache/kylin/query/optrule/OLAPJoinRule.java @@ -63,7 +63,7 @@ public class OLAPJoinRule extends ConverterRule { try { newRel = new OLAPJoinRel(cluster, traitSet, left, right, // info.getEquiCondition(left, right, cluster.getRexBuilder()), // - info.leftKeys, info.rightKeys, join.getJoinType(), join.getVariablesStopped()); + info.leftKeys, info.rightKeys, join.getVariablesSet(), join.getJoinType()); } catch (InvalidRelException e) { // Semantic error not possible. Must be a bug. Convert to internal error. throw new AssertionError(e); diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java index eed5636..9414757 100644 --- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java +++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java @@ -37,6 +37,7 @@ import org.apache.calcite.rel.InvalidRelException; import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.core.Aggregate; import org.apache.calcite.rel.core.AggregateCall; +import org.apache.calcite.rel.metadata.RelMetadataQuery; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeFactory; import org.apache.calcite.rel.type.RelDataTypeField; @@ -60,6 +61,7 @@ import org.apache.kylin.metadata.model.MeasureDesc; import org.apache.kylin.metadata.model.ParameterDesc; import org.apache.kylin.metadata.model.TableDesc; import org.apache.kylin.metadata.model.TblColRef; +import org.apache.kylin.query.schema.OLAPTable; import com.google.common.base.Preconditions; import com.google.common.collect.Lists; @@ -122,8 +124,8 @@ public class OLAPAggregateRel extends Aggregate implements OLAPRel { } @Override - public RelOptCost computeSelfCost(RelOptPlanner planner) { - return super.computeSelfCost(planner).multiplyBy(.05); + public RelOptCost computeSelfCost(RelOptPlanner planner, RelMetadataQuery mq) { + return super.computeSelfCost(planner, mq).multiplyBy(.05); } @Override @@ -280,6 +282,7 @@ public class OLAPAggregateRel extends Aggregate implements OLAPRel { fillbackOptimizedColumn(); ColumnRowType inputColumnRowType = ((OLAPRel) getInput()).getColumnRowType(); + RelDataTypeFactory typeFactory = getCluster().getTypeFactory(); for (int i = 0; i < this.aggregations.size(); i++) { FunctionDesc aggFunc = this.aggregations.get(i); @@ -290,7 +293,8 @@ public class OLAPAggregateRel extends Aggregate implements OLAPRel { if (aggFunc.needRewrite()) { String rewriteFieldName = aggFunc.getRewriteFieldName(); - this.context.rewriteFields.put(rewriteFieldName, null); + RelDataType rewriteFieldType = OLAPTable.createSqlType(typeFactory, aggFunc.getRewriteFieldType(), true); + this.context.rewriteFields.put(rewriteFieldName, rewriteFieldType); TblColRef column = buildRewriteColumn(aggFunc); this.context.metricsColumns.add(column); diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java index a847890..5ea138f 100644 --- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java +++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java @@ -36,6 +36,7 @@ import org.apache.calcite.plan.RelTrait; import org.apache.calcite.plan.RelTraitSet; import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.core.Filter; +import org.apache.calcite.rel.metadata.RelMetadataQuery; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rex.RexBuilder; import org.apache.calcite.rex.RexCall; @@ -252,8 +253,8 @@ public class OLAPFilterRel extends Filter implements OLAPRel { } @Override - public RelOptCost computeSelfCost(RelOptPlanner planner) { - return super.computeSelfCost(planner).multiplyBy(.05); + public RelOptCost computeSelfCost(RelOptPlanner planner, RelMetadataQuery mq) { + return super.computeSelfCost(planner, mq).multiplyBy(.05); } @Override diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPJoinRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPJoinRel.java index 09a0b17..efe404b 100644 --- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPJoinRel.java +++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPJoinRel.java @@ -43,8 +43,10 @@ import org.apache.calcite.plan.RelTrait; import org.apache.calcite.plan.RelTraitSet; import org.apache.calcite.rel.InvalidRelException; import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.core.CorrelationId; import org.apache.calcite.rel.core.JoinInfo; import org.apache.calcite.rel.core.JoinRelType; +import org.apache.calcite.rel.metadata.RelMetadataQuery; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeFactory.FieldInfoBuilder; import org.apache.calcite.rel.type.RelDataTypeField; @@ -73,8 +75,8 @@ public class OLAPJoinRel extends EnumerableJoin implements OLAPRel { public OLAPJoinRel(RelOptCluster cluster, RelTraitSet traits, RelNode left, RelNode right, // RexNode condition, ImmutableIntList leftKeys, ImmutableIntList rightKeys, // - JoinRelType joinType, Set variablesStopped) throws InvalidRelException { - super(cluster, traits, left, right, condition, leftKeys, rightKeys, joinType, variablesStopped); + Set variablesSet, JoinRelType joinType) throws InvalidRelException { + super(cluster, traits, left, right, condition, leftKeys, rightKeys, variablesSet, joinType); Preconditions.checkArgument(getConvention() == OLAPRel.CONVENTION); this.rowType = getRowType(); this.isTopJoin = false; @@ -87,7 +89,7 @@ public class OLAPJoinRel extends EnumerableJoin implements OLAPRel { final JoinInfo joinInfo = JoinInfo.of(left, right, condition); assert joinInfo.isEqui(); try { - return new OLAPJoinRel(getCluster(), traitSet, left, right, condition, joinInfo.leftKeys, joinInfo.rightKeys, joinType, variablesStopped); + return new OLAPJoinRel(getCluster(), traitSet, left, right, condition, joinInfo.leftKeys, joinInfo.rightKeys, variablesSet, joinType); } catch (InvalidRelException e) { // Semantic error not possible. Must be a bug. Convert to internal error. throw new AssertionError(e); @@ -95,13 +97,13 @@ public class OLAPJoinRel extends EnumerableJoin implements OLAPRel { } @Override - public RelOptCost computeSelfCost(RelOptPlanner planner) { - return super.computeSelfCost(planner).multiplyBy(.05); + public RelOptCost computeSelfCost(RelOptPlanner planner, RelMetadataQuery mq) { + return super.computeSelfCost(planner, mq).multiplyBy(.05); } @Override - public double getRows() { - return super.getRows() * 0.1; + public double estimateRowCount(RelMetadataQuery mq) { + return super.estimateRowCount(mq) * 0.1; } @Override @@ -225,8 +227,8 @@ public class OLAPJoinRel extends EnumerableJoin implements OLAPRel { RexNode.class, // ImmutableIntList.class, // ImmutableIntList.class, // - JoinRelType.class, // - Set.class); + Set.class, // + JoinRelType.class); constr.setAccessible(true); } catch (Exception e) { throw new RuntimeException(e); @@ -238,7 +240,7 @@ public class OLAPJoinRel extends EnumerableJoin implements OLAPRel { if (this.hasSubQuery) { try { return constr.newInstance(getCluster(), getCluster().traitSetOf(EnumerableConvention.INSTANCE), // - inputs.get(0), inputs.get(1), condition, leftKeys, rightKeys, joinType, variablesStopped); + inputs.get(0), inputs.get(1), condition, leftKeys, rightKeys, variablesSet, joinType); } catch (Exception e) { throw new IllegalStateException("Can't create EnumerableJoin!", e); } diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPLimitRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPLimitRel.java index 82aa9de..988d1fa 100644 --- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPLimitRel.java +++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPLimitRel.java @@ -31,6 +31,7 @@ import org.apache.calcite.plan.RelTraitSet; import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.RelWriter; import org.apache.calcite.rel.SingleRel; +import org.apache.calcite.rel.metadata.RelMetadataQuery; import org.apache.calcite.rex.RexLiteral; import org.apache.calcite.rex.RexNode; @@ -59,8 +60,8 @@ public class OLAPLimitRel extends SingleRel implements OLAPRel { } @Override - public RelOptCost computeSelfCost(RelOptPlanner planner) { - return super.computeSelfCost(planner).multiplyBy(.05); + public RelOptCost computeSelfCost(RelOptPlanner planner, RelMetadataQuery mq) { + return super.computeSelfCost(planner, mq).multiplyBy(.05); } @Override diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPProjectRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPProjectRel.java index b0436e2..8e454c9 100644 --- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPProjectRel.java +++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPProjectRel.java @@ -35,6 +35,7 @@ import org.apache.calcite.plan.RelTrait; import org.apache.calcite.plan.RelTraitSet; import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.core.Project; +import org.apache.calcite.rel.metadata.RelMetadataQuery; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeFactory.FieldInfoBuilder; import org.apache.calcite.rel.type.RelDataTypeField; @@ -86,8 +87,8 @@ public class OLAPProjectRel extends Project implements OLAPRel { } @Override - public RelOptCost computeSelfCost(RelOptPlanner planner) { - return super.computeSelfCost(planner).multiplyBy(.05); + public RelOptCost computeSelfCost(RelOptPlanner planner, RelMetadataQuery mq) { + return super.computeSelfCost(planner, mq).multiplyBy(.05); } @Override diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPSortRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPSortRel.java index c3e0595..64a6c3c 100644 --- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPSortRel.java +++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPSortRel.java @@ -32,6 +32,7 @@ import org.apache.calcite.rel.RelCollation; import org.apache.calcite.rel.RelFieldCollation; import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.core.Sort; +import org.apache.calcite.rel.metadata.RelMetadataQuery; import org.apache.calcite.rex.RexNode; import org.apache.kylin.metadata.model.MeasureDesc; import org.apache.kylin.metadata.model.TblColRef; @@ -58,8 +59,8 @@ public class OLAPSortRel extends Sort implements OLAPRel { } @Override - public RelOptCost computeSelfCost(RelOptPlanner planner) { - return super.computeSelfCost(planner).multiplyBy(.05); + public RelOptCost computeSelfCost(RelOptPlanner planner, RelMetadataQuery mq) { + return super.computeSelfCost(planner, mq).multiplyBy(.05); } @Override diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java index 675fd99..bcef413 100644 --- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java +++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java @@ -40,14 +40,17 @@ import org.apache.calcite.plan.volcano.AbstractConverter.ExpandConversionRule; import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.RelWriter; import org.apache.calcite.rel.core.TableScan; +import org.apache.calcite.rel.metadata.RelMetadataQuery; import org.apache.calcite.rel.rules.AggregateExpandDistinctAggregatesRule; import org.apache.calcite.rel.rules.AggregateJoinTransposeRule; import org.apache.calcite.rel.rules.AggregateProjectMergeRule; import org.apache.calcite.rel.rules.FilterJoinRule; import org.apache.calcite.rel.rules.FilterProjectTransposeRule; import org.apache.calcite.rel.rules.JoinCommuteRule; +import org.apache.calcite.rel.rules.JoinPushExpressionsRule; import org.apache.calcite.rel.rules.JoinPushThroughJoinRule; import org.apache.calcite.rel.rules.ReduceExpressionsRule; +import org.apache.calcite.rel.rules.SortJoinTransposeRule; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeFactory; import org.apache.calcite.rel.type.RelDataTypeField; @@ -147,6 +150,8 @@ public class OLAPTableScan extends TableScan implements OLAPRel, EnumerableRel { planner.removeRule(AggregateJoinTransposeRule.INSTANCE); planner.removeRule(AggregateProjectMergeRule.INSTANCE); planner.removeRule(FilterProjectTransposeRule.INSTANCE); + planner.removeRule(SortJoinTransposeRule.INSTANCE); + planner.removeRule(JoinPushExpressionsRule.INSTANCE); // distinct count will be split into a separated query that is joined with the left query planner.removeRule(AggregateExpandDistinctAggregatesRule.INSTANCE); @@ -165,8 +170,8 @@ public class OLAPTableScan extends TableScan implements OLAPRel, EnumerableRel { } @Override - public RelOptCost computeSelfCost(RelOptPlanner planner) { - return super.computeSelfCost(planner).multiplyBy(.05); + public RelOptCost computeSelfCost(RelOptPlanner planner, RelMetadataQuery mq) { + return super.computeSelfCost(planner, mq).multiplyBy(.05); } @Override diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPToEnumerableConverter.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPToEnumerableConverter.java index 7053694..de7e7e2 100644 --- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPToEnumerableConverter.java +++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPToEnumerableConverter.java @@ -35,6 +35,7 @@ import org.apache.calcite.plan.RelOptUtil; import org.apache.calcite.plan.RelTraitSet; import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.convert.ConverterImpl; +import org.apache.calcite.rel.metadata.RelMetadataQuery; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.sql.SqlExplainLevel; import org.apache.kylin.metadata.realization.IRealization; @@ -56,8 +57,9 @@ public class OLAPToEnumerableConverter extends ConverterImpl implements Enumerab } @Override - public RelOptCost computeSelfCost(RelOptPlanner planner) { - return super.computeSelfCost(planner).multiplyBy(.05); + public RelOptCost computeSelfCost(RelOptPlanner planner, RelMetadataQuery mq) { + // huge cost to ensure OLAPToEnumerableConverter only appears once in rel tree + return planner.getCostFactory().makeCost(1E100, 0, 0); } @Override diff --git a/query/src/main/java/org/apache/kylin/query/schema/OLAPTable.java b/query/src/main/java/org/apache/kylin/query/schema/OLAPTable.java index a8789ea..12ed6e5 100644 --- a/query/src/main/java/org/apache/kylin/query/schema/OLAPTable.java +++ b/query/src/main/java/org/apache/kylin/query/schema/OLAPTable.java @@ -43,6 +43,7 @@ import org.apache.calcite.schema.impl.AbstractTableQueryable; import org.apache.calcite.sql.type.SqlTypeName; import org.apache.calcite.sql.type.SqlTypeUtil; import org.apache.calcite.util.ImmutableBitSet; +import org.apache.kylin.metadata.datatype.DataType; import org.apache.kylin.metadata.model.ColumnDesc; import org.apache.kylin.metadata.model.FunctionDesc; import org.apache.kylin.metadata.model.MeasureDesc; @@ -128,21 +129,21 @@ public class OLAPTable extends AbstractQueryableTable implements TranslatableTab private RelDataType deriveRowType(RelDataTypeFactory typeFactory) { RelDataTypeFactory.FieldInfoBuilder fieldInfo = typeFactory.builder(); for (ColumnDesc column : exposedColumns) { - RelDataType sqlType = createSqlType(typeFactory, column); + RelDataType sqlType = createSqlType(typeFactory, column.getType(), column.isNullable()); sqlType = SqlTypeUtil.addCharsetAndCollation(sqlType, typeFactory); fieldInfo.add(column.getName(), sqlType); } return typeFactory.createStructType(fieldInfo); } - private RelDataType createSqlType(RelDataTypeFactory typeFactory, ColumnDesc column) { - SqlTypeName sqlTypeName = SQLTYPE_MAPPING.get(column.getTypeName()); + public static RelDataType createSqlType(RelDataTypeFactory typeFactory, DataType dataType, boolean isNullable) { + SqlTypeName sqlTypeName = SQLTYPE_MAPPING.get(dataType.getName()); if (sqlTypeName == null) - throw new IllegalArgumentException("Unrecognized column type " + column.getTypeName() + " from " + column); - - int precision = column.getTypePrecision(); - int scale = column.getTypeScale(); + throw new IllegalArgumentException("Unrecognized data type " + dataType); + int precision = dataType.getPrecision(); + int scale = dataType.getScale(); + RelDataType result; if (precision >= 0 && scale >= 0) result = typeFactory.createSqlType(sqlTypeName, precision, scale); @@ -152,7 +153,7 @@ public class OLAPTable extends AbstractQueryableTable implements TranslatableTab result = typeFactory.createSqlType(sqlTypeName); // due to left join and uncertain data quality, dimension value can be null - if (column.isNullable()) { + if (isNullable) { result = typeFactory.createTypeWithNullability(result, true); } else { result = typeFactory.createTypeWithNullability(result, false); -- 1.9.5.msysgit.0