From 7e0e1cb652750eca0effeb9569f5a2f47ed6a5f7 Mon Sep 17 00:00:00 2001 From: jiazhong Date: Fri, 28 Nov 2014 15:17:56 +0800 Subject: [PATCH 01/35] correct the cube size & job size in job&cubes page --- webapp/app/js/controllers/cubes.js | 2 +- webapp/app/js/controllers/job.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/webapp/app/js/controllers/cubes.js b/webapp/app/js/controllers/cubes.js index 423e783..edffc56 100644 --- a/webapp/app/js/controllers/cubes.js +++ b/webapp/app/js/controllers/cubes.js @@ -75,7 +75,7 @@ KylinApp $scope.$watch('project.selectedProject', function (newValue, oldValue) { if(newValue){ $scope.cubes=[]; - $scope.list(); + $scope.reload(); } }); diff --git a/webapp/app/js/controllers/job.js b/webapp/app/js/controllers/job.js index 345b2b3..84afd1e 100644 --- a/webapp/app/js/controllers/job.js +++ b/webapp/app/js/controllers/job.js @@ -85,7 +85,7 @@ KylinApp if(newValue){ $scope.jobs={}; $scope.state.projectName = newValue; - $scope.list(); + $scope.reload(); } }); From edf5c8b57c76c8d928223abc6b5375c8f8bd542d Mon Sep 17 00:00:00 2001 From: jiazhong Date: Fri, 28 Nov 2014 16:24:42 +0800 Subject: [PATCH 02/35] rm auto reloas --- server/src/main/java/com/kylinolap/rest/controller/CubeController.java | 2 +- webapp/app/partials/cubes/cubes.html | 2 +- webapp/app/partials/jobs/jobs.html | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/com/kylinolap/rest/controller/CubeController.java b/server/src/main/java/com/kylinolap/rest/controller/CubeController.java index 839d36b..5f6f214 100644 --- a/server/src/main/java/com/kylinolap/rest/controller/CubeController.java +++ b/server/src/main/java/com/kylinolap/rest/controller/CubeController.java @@ -315,7 +315,7 @@ public CubeRequest saveCubeDesc(@RequestBody CubeRequest cubeRequest) { cubeService.createCubeAndDesc(name, projectName, desc); } catch (Exception e) { logger.error("Failed to deal with the request.", e); - throw new InternalErrorException(e.getLocalizedMessage()); + throw new InternalErrorException(e.getLocalizedMessage(),e); } cubeRequest.setUuid(desc.getUuid()); diff --git a/webapp/app/partials/cubes/cubes.html b/webapp/app/partials/cubes/cubes.html index 5a689a8..aff17d5 100644 --- a/webapp/app/partials/cubes/cubes.html +++ b/webapp/app/partials/cubes/cubes.html @@ -130,7 +130,7 @@
- +
diff --git a/webapp/app/partials/jobs/jobs.html b/webapp/app/partials/jobs/jobs.html index 4a1522f..faf02e0 100644 --- a/webapp/app/partials/jobs/jobs.html +++ b/webapp/app/partials/jobs/jobs.html @@ -138,7 +138,7 @@
- +
From 64092dccd2b9cca89f01b53f77e51c36ca8566c8 Mon Sep 17 00:00:00 2001 From: Luke Han Date: Fri, 28 Nov 2014 16:27:33 +0800 Subject: [PATCH 03/35] [maven-release-plugin] prepare release kylin-0.6.3 --- common/pom.xml | 2 +- cube/pom.xml | 2 +- dictionary/pom.xml | 156 ++++---- jdbc/pom.xml | 2 +- job/pom.xml | 2 +- metadata/pom.xml | 2 +- pom.xml | 1082 ++++++++++++++++++++++++++-------------------------- query/pom.xml | 2 +- server/pom.xml | 2 +- storage/pom.xml | 232 +++++------ 10 files changed, 742 insertions(+), 742 deletions(-) diff --git a/common/pom.xml b/common/pom.xml index 654df7e..4c36750 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -8,7 +8,7 @@ com.kylinolap kylin - 0.6.3-SNAPSHOT + 0.6.3 diff --git a/cube/pom.xml b/cube/pom.xml index f177c07..490a817 100644 --- a/cube/pom.xml +++ b/cube/pom.xml @@ -7,7 +7,7 @@ com.kylinolap kylin - 0.6.3-SNAPSHOT + 0.6.3 diff --git a/dictionary/pom.xml b/dictionary/pom.xml index f345b44..be1ced3 100644 --- a/dictionary/pom.xml +++ b/dictionary/pom.xml @@ -1,78 +1,78 @@ - - 4.0.0 - - kylin-dictionary - jar - Kylin:Dictionary - - - com.kylinolap - kylin - 0.6.3-SNAPSHOT - - - - - - - - - com.kylinolap - kylin-metadata - ${project.parent.version} - - - - com.fasterxml.jackson.core - jackson-databind - - - commons-io - commons-io - - - commons-configuration - commons-configuration - - - com.google.guava - guava - - - - - - org.apache.hadoop - hadoop-common - provided - - - org.apache.hadoop - hadoop-hdfs - provided - - - - com.google.protobuf - protobuf-java - - - - - org.apache.hbase - hbase-common - provided - - - org.apache.hbase - hbase-client - provided - - - junit - junit - test - - - - + + 4.0.0 + + kylin-dictionary + jar + Kylin:Dictionary + + + com.kylinolap + kylin + 0.6.3 + + + + + + + + + com.kylinolap + kylin-metadata + ${project.parent.version} + + + + com.fasterxml.jackson.core + jackson-databind + + + commons-io + commons-io + + + commons-configuration + commons-configuration + + + com.google.guava + guava + + + + + + org.apache.hadoop + hadoop-common + provided + + + org.apache.hadoop + hadoop-hdfs + provided + + + + com.google.protobuf + protobuf-java + + + + + org.apache.hbase + hbase-common + provided + + + org.apache.hbase + hbase-client + provided + + + junit + junit + test + + + + diff --git a/jdbc/pom.xml b/jdbc/pom.xml index e69cac0..4fc37a7 100644 --- a/jdbc/pom.xml +++ b/jdbc/pom.xml @@ -11,7 +11,7 @@ com.kylinolap kylin - 0.6.3-SNAPSHOT + 0.6.3 diff --git a/job/pom.xml b/job/pom.xml index b039957..05cb6ad 100644 --- a/job/pom.xml +++ b/job/pom.xml @@ -4,7 +4,7 @@ com.kylinolap kylin - 0.6.3-SNAPSHOT + 0.6.3 kylin-job diff --git a/metadata/pom.xml b/metadata/pom.xml index 3e76619..a77c88a 100644 --- a/metadata/pom.xml +++ b/metadata/pom.xml @@ -8,7 +8,7 @@ com.kylinolap kylin - 0.6.3-SNAPSHOT + 0.6.3 diff --git a/pom.xml b/pom.xml index a247347..fc67b93 100644 --- a/pom.xml +++ b/pom.xml @@ -1,541 +1,541 @@ - - - 4.0.0 - com.kylinolap - kylin - pom - 0.6.3-SNAPSHOT - Kylin:HadoopOLAPEngine - - - - 1.7 - 3.1.1 - 2.5.1 - UTF-8 - UTF-8 - - - 2.4.1 - 2.4.1 - 0.98.0-hadoop2 - 3.4.5 - 0.13.0 - - - 3.4 - 4.11 - 1.0.0 - 2.5.0 - 1.3.174 - - - 1.2 - 2.6 - 3.1 - 2.4 - 1.9 - 1.0.15 - 3.1 - - - 1.2.17 - 1.6.4 - 2.2.3 - 12.0.1 - 0.1.51 - 2.9.1 - 2.7.1 - 0.8.4 - 1.3.4 - - - 3.1.2.RELEASE - - - 0.9.1-incubating - 0.4 - - - 3.0.1 - - - 2.2.1 - - - 2.6.0 - - - jacoco - reuseReports - ${project.basedir}/../target/jacoco.exec - java - com/kylinolap/**/tools/**:net/hydromatic/optiq/**:org/eigenbase/sql2rel/** - - - - - - - org.apache.hadoop - hadoop-common - ${hadoop2.version} - compile - - - javax.servlet - servlet-api - - - - - org.apache.hadoop - hadoop-hdfs - ${hadoop2.version} - - - org.apache.hadoop - hadoop-mapreduce-client-app - ${hadoop2.version} - - - org.apache.hadoop - hadoop-yarn-api - ${hadoop2.version} - - - org.apache.hadoop - hadoop-mapreduce-client-core - ${hadoop2.version} - - - org.apache.hadoop - hadoop-mapreduce-client-jobclient - ${hadoop2.version} - - - org.apache.hadoop - hadoop-annotations - ${hadoop2.version} - - - org.apache.hadoop - hadoop-auth - ${hadoop2.version} - - - org.apache.hadoop - hadoop-minicluster - ${hadoop2.version} - true - - - - - org.apache.hbase - hbase-hadoop2-compat - ${hbase-hadoop2.version} - - - org.apache.hbase - hbase-common - ${hbase-hadoop2.version} - - - org.apache.hbase - hbase-client - ${hbase-hadoop2.version} - - - org.apache.hbase - hbase-server - ${hbase-hadoop2.version} - - - org.apache.mrunit - mrunit - ${mrunit.version} - hadoop2 - - - - - org.apache.hive - hive-jdbc - ${hive.version} - - - - - org.apache.hadoop - hadoop-yarn-server-resourcemanager - ${yarn.version} - - - - - org.apache.calcite - calcite-core - ${optiq.version} - - - org.apache.calcite - calcite-avatica - ${optiq.version} - - - net.hydromatic - linq4j - ${linq4j.version} - - - - - junit - junit - ${junit.version} - - - org.apache.zookeeper - zookeeper - ${zookeeper.version} - - - commons-cli - commons-cli - ${commons-cli.version} - - - commons-lang - commons-lang - ${commons-lang.version} - - - org.apache.commons - commons-lang3 - ${commons-lang3.version} - - - commons-io - commons-io - ${commons-io.version} - - - commons-configuration - commons-configuration - ${commons-configuration.version} - - - commons-daemon - commons-daemon - ${commons-daemon.version} - - - log4j - log4j - ${log4j.version} - - - org.slf4j - jcl-over-slf4j - ${slf4j.version} - - - org.slf4j - slf4j-api - ${slf4j.version} - - - org.slf4j - slf4j-log4j12 - ${slf4j.version} - - - com.fasterxml.jackson.core - jackson-databind - ${jackson.version} - - - com.fasterxml.jackson.core - jackson-core - ${jackson.version} - - - commons-httpclient - commons-httpclient - ${commons-httpclient.version} - - - com.google.guava - guava - ${guava.version} - - - com.jcraft - jsch - ${jsch.version} - - - org.dbunit - dbunit - ${dbunit.version} - - - org.apache.maven - maven-model - ${maven-model.version} - - - com.h2database - h2 - ${h2.version} - - - xerces - xercesImpl - ${xerces.version} - - - xalan - xalan - ${xalan.version} - - - com.ning - compress-lzf - ${compress-lzf.version} - - - com.n3twork.druid - extendedset - ${extendedset.version} - - - - org.quartz-scheduler - quartz - ${quartz.version} - - - org.quartz-scheduler - quartz-jobs - ${quartz.version} - - - - org.apache.curator - curator-framework - ${curator.version} - - - org.apache.curator - curator-recipes - ${curator.version} - - - - - - - - central - http://repo.maven.apache.org/maven2 - - - conjars - http://conjars.org/repo/ - - - - true - always - warn - - - false - never - fail - - HDPReleases - HDP Releases - http://repo.hortonworks.com/content/repositories/releases/ - default - - - - - - - - org.apache.maven.plugins - maven-compiler-plugin - 3.1 - - ${javaVersion} - ${javaVersion} - - - - org.apache.maven.plugins - maven-site-plugin - 2.0 - - - org.apache.maven.plugins - maven-install-plugin - 2.2 - - - org.apache.maven.plugins - maven-resources-plugin - 2.4 - - UTF-8 - - - - org.apache.maven.plugins - maven-jar-plugin - 2.4 - - - - test-jar - - - - - - org.apache.maven.plugins - maven-war-plugin - 2.0.2 - - - org.apache.maven.plugins - maven-dependency-plugin - - - org.apache.maven.plugins - maven-antrun-plugin - - - org.apache.maven.plugins - maven-source-plugin - 2.1.2 - - - attach-sources - package - - jar-no-fork - - - - - true - - - - org.antlr - antlr3-maven-plugin - ${antlr.version} - - - - - antlr - - false - - - - - - org.apache.maven.plugins - maven-failsafe-plugin - 2.6 - - - - integration-test - verify - - - - - - org.apache.maven.plugins - maven-release-plugin - ${maven-release.version} - - - org.jacoco - jacoco-maven-plugin - 0.7.0.201403182114 - - ${sonar.jacoco.reportPath} - true - - - - agent - - prepare-agent - - - - - - - - - - scm:git:git@github.com:KylinOLAP/Kylin.git - scm:git:git@github.com:KylinOLAP/Kylin.git - scm:git:git@github.com:KylinOLAP/Kylin.git - HEAD - - - - common - metadata - dictionary - cube - job - storage - query - server - jdbc - - - - default - - true - - - - - org.apache.maven.plugins - maven-surefire-plugin - 2.16 - - - **/com/kylinolap/job/** - **/BuildCubeWithEngineTest.java - **/SampleCubeSetupTest.java - **/InvertedIndexHBaseTest.java - **/StorageTest.java - **/KylinQueryTest.java - - - - - - - - - e2e - - - + + + 4.0.0 + com.kylinolap + kylin + pom + 0.6.3 + Kylin:HadoopOLAPEngine + + + + 1.7 + 3.1.1 + 2.5.1 + UTF-8 + UTF-8 + + + 2.4.1 + 2.4.1 + 0.98.0-hadoop2 + 3.4.5 + 0.13.0 + + + 3.4 + 4.11 + 1.0.0 + 2.5.0 + 1.3.174 + + + 1.2 + 2.6 + 3.1 + 2.4 + 1.9 + 1.0.15 + 3.1 + + + 1.2.17 + 1.6.4 + 2.2.3 + 12.0.1 + 0.1.51 + 2.9.1 + 2.7.1 + 0.8.4 + 1.3.4 + + + 3.1.2.RELEASE + + + 0.9.1-incubating + 0.4 + + + 3.0.1 + + + 2.2.1 + + + 2.6.0 + + + jacoco + reuseReports + ${project.basedir}/../target/jacoco.exec + java + com/kylinolap/**/tools/**:net/hydromatic/optiq/**:org/eigenbase/sql2rel/** + + + + + + + org.apache.hadoop + hadoop-common + ${hadoop2.version} + compile + + + javax.servlet + servlet-api + + + + + org.apache.hadoop + hadoop-hdfs + ${hadoop2.version} + + + org.apache.hadoop + hadoop-mapreduce-client-app + ${hadoop2.version} + + + org.apache.hadoop + hadoop-yarn-api + ${hadoop2.version} + + + org.apache.hadoop + hadoop-mapreduce-client-core + ${hadoop2.version} + + + org.apache.hadoop + hadoop-mapreduce-client-jobclient + ${hadoop2.version} + + + org.apache.hadoop + hadoop-annotations + ${hadoop2.version} + + + org.apache.hadoop + hadoop-auth + ${hadoop2.version} + + + org.apache.hadoop + hadoop-minicluster + ${hadoop2.version} + true + + + + + org.apache.hbase + hbase-hadoop2-compat + ${hbase-hadoop2.version} + + + org.apache.hbase + hbase-common + ${hbase-hadoop2.version} + + + org.apache.hbase + hbase-client + ${hbase-hadoop2.version} + + + org.apache.hbase + hbase-server + ${hbase-hadoop2.version} + + + org.apache.mrunit + mrunit + ${mrunit.version} + hadoop2 + + + + + org.apache.hive + hive-jdbc + ${hive.version} + + + + + org.apache.hadoop + hadoop-yarn-server-resourcemanager + ${yarn.version} + + + + + org.apache.calcite + calcite-core + ${optiq.version} + + + org.apache.calcite + calcite-avatica + ${optiq.version} + + + net.hydromatic + linq4j + ${linq4j.version} + + + + + junit + junit + ${junit.version} + + + org.apache.zookeeper + zookeeper + ${zookeeper.version} + + + commons-cli + commons-cli + ${commons-cli.version} + + + commons-lang + commons-lang + ${commons-lang.version} + + + org.apache.commons + commons-lang3 + ${commons-lang3.version} + + + commons-io + commons-io + ${commons-io.version} + + + commons-configuration + commons-configuration + ${commons-configuration.version} + + + commons-daemon + commons-daemon + ${commons-daemon.version} + + + log4j + log4j + ${log4j.version} + + + org.slf4j + jcl-over-slf4j + ${slf4j.version} + + + org.slf4j + slf4j-api + ${slf4j.version} + + + org.slf4j + slf4j-log4j12 + ${slf4j.version} + + + com.fasterxml.jackson.core + jackson-databind + ${jackson.version} + + + com.fasterxml.jackson.core + jackson-core + ${jackson.version} + + + commons-httpclient + commons-httpclient + ${commons-httpclient.version} + + + com.google.guava + guava + ${guava.version} + + + com.jcraft + jsch + ${jsch.version} + + + org.dbunit + dbunit + ${dbunit.version} + + + org.apache.maven + maven-model + ${maven-model.version} + + + com.h2database + h2 + ${h2.version} + + + xerces + xercesImpl + ${xerces.version} + + + xalan + xalan + ${xalan.version} + + + com.ning + compress-lzf + ${compress-lzf.version} + + + com.n3twork.druid + extendedset + ${extendedset.version} + + + + org.quartz-scheduler + quartz + ${quartz.version} + + + org.quartz-scheduler + quartz-jobs + ${quartz.version} + + + + org.apache.curator + curator-framework + ${curator.version} + + + org.apache.curator + curator-recipes + ${curator.version} + + + + + + + + central + http://repo.maven.apache.org/maven2 + + + conjars + http://conjars.org/repo/ + + + + true + always + warn + + + false + never + fail + + HDPReleases + HDP Releases + http://repo.hortonworks.com/content/repositories/releases/ + default + + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.1 + + ${javaVersion} + ${javaVersion} + + + + org.apache.maven.plugins + maven-site-plugin + 2.0 + + + org.apache.maven.plugins + maven-install-plugin + 2.2 + + + org.apache.maven.plugins + maven-resources-plugin + 2.4 + + UTF-8 + + + + org.apache.maven.plugins + maven-jar-plugin + 2.4 + + + + test-jar + + + + + + org.apache.maven.plugins + maven-war-plugin + 2.0.2 + + + org.apache.maven.plugins + maven-dependency-plugin + + + org.apache.maven.plugins + maven-antrun-plugin + + + org.apache.maven.plugins + maven-source-plugin + 2.1.2 + + + attach-sources + package + + jar-no-fork + + + + + true + + + + org.antlr + antlr3-maven-plugin + ${antlr.version} + + + + + antlr + + false + + + + + + org.apache.maven.plugins + maven-failsafe-plugin + 2.6 + + + + integration-test + verify + + + + + + org.apache.maven.plugins + maven-release-plugin + ${maven-release.version} + + + org.jacoco + jacoco-maven-plugin + 0.7.0.201403182114 + + ${sonar.jacoco.reportPath} + true + + + + agent + + prepare-agent + + + + + + + + + + scm:git:git@github.com:KylinOLAP/Kylin.git + scm:git:git@github.com:KylinOLAP/Kylin.git + scm:git:git@github.com:KylinOLAP/Kylin.git + kylin-0.6.3 + + + + common + metadata + dictionary + cube + job + storage + query + server + jdbc + + + + default + + true + + + + + org.apache.maven.plugins + maven-surefire-plugin + 2.16 + + + **/com/kylinolap/job/** + **/BuildCubeWithEngineTest.java + **/SampleCubeSetupTest.java + **/InvertedIndexHBaseTest.java + **/StorageTest.java + **/KylinQueryTest.java + + + + + + + + + e2e + + + diff --git a/query/pom.xml b/query/pom.xml index 189eb82..d7ca6b0 100644 --- a/query/pom.xml +++ b/query/pom.xml @@ -9,7 +9,7 @@ com.kylinolap kylin - 0.6.3-SNAPSHOT + 0.6.3 diff --git a/server/pom.xml b/server/pom.xml index fea3397..9315dfe 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -9,7 +9,7 @@ com.kylinolap kylin - 0.6.3-SNAPSHOT + 0.6.3 diff --git a/storage/pom.xml b/storage/pom.xml index 3ea55a7..520fd0e 100644 --- a/storage/pom.xml +++ b/storage/pom.xml @@ -1,117 +1,117 @@ - - 4.0.0 - - kylin-storage - Kylin:Storage - - - com.kylinolap - kylin - 0.6.3-SNAPSHOT - - - - - com.kylinolap - kylin-cube - ${project.parent.version} - - - - - - org.apache.hbase - hbase-client - provided - - - org.apache.hbase - hbase-server - provided - - - org.apache.hadoop - hadoop-hdfs - provided - - - - com.google.protobuf - protobuf-java - - - - - - junit - junit - test - - - org.apache.hbase - hbase-testing-util - ${hbase-hadoop2.version} - test - - - javax.servlet - servlet-api - - - javax.servlet.jsp - jsp-api - - - - - - - - - org.apache.maven.plugins - maven-shade-plugin - 2.2 - - - package - - shade - - - true - coprocessor - - - - - - com.kylinolap:kylin-common - com.kylinolap:kylin-metadata - com.kylinolap:kylin-dictionary - com.kylinolap:kylin-cube - com.kylinolap:kylin-storage - net.sf.trove4j:* - - - - - - - - - *:* - - META-INF/*.SF - META-INF/*.DSA - META-INF/*.RSA - - - - - - - - - - + + 4.0.0 + + kylin-storage + Kylin:Storage + + + com.kylinolap + kylin + 0.6.3 + + + + + com.kylinolap + kylin-cube + ${project.parent.version} + + + + + + org.apache.hbase + hbase-client + provided + + + org.apache.hbase + hbase-server + provided + + + org.apache.hadoop + hadoop-hdfs + provided + + + + com.google.protobuf + protobuf-java + + + + + + junit + junit + test + + + org.apache.hbase + hbase-testing-util + ${hbase-hadoop2.version} + test + + + javax.servlet + servlet-api + + + javax.servlet.jsp + jsp-api + + + + + + + + + org.apache.maven.plugins + maven-shade-plugin + 2.2 + + + package + + shade + + + true + coprocessor + + + + + + com.kylinolap:kylin-common + com.kylinolap:kylin-metadata + com.kylinolap:kylin-dictionary + com.kylinolap:kylin-cube + com.kylinolap:kylin-storage + net.sf.trove4j:* + + + + + + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + + + + + \ No newline at end of file From decbdb006ccead4b772cb5acb4786d6d3c55dd52 Mon Sep 17 00:00:00 2001 From: Luke Han Date: Fri, 28 Nov 2014 16:40:32 +0800 Subject: [PATCH 04/35] [maven-release-plugin] rollback the release of kylin-0.6.3 --- common/pom.xml | 2 +- cube/pom.xml | 2 +- dictionary/pom.xml | 2 +- jdbc/pom.xml | 2 +- job/pom.xml | 2 +- metadata/pom.xml | 2 +- pom.xml | 4 ++-- query/pom.xml | 2 +- server/pom.xml | 2 +- storage/pom.xml | 2 +- 10 files changed, 11 insertions(+), 11 deletions(-) diff --git a/common/pom.xml b/common/pom.xml index 4c36750..654df7e 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -8,7 +8,7 @@ com.kylinolap kylin - 0.6.3 + 0.6.3-SNAPSHOT diff --git a/cube/pom.xml b/cube/pom.xml index 490a817..f177c07 100644 --- a/cube/pom.xml +++ b/cube/pom.xml @@ -7,7 +7,7 @@ com.kylinolap kylin - 0.6.3 + 0.6.3-SNAPSHOT diff --git a/dictionary/pom.xml b/dictionary/pom.xml index be1ced3..1d09903 100644 --- a/dictionary/pom.xml +++ b/dictionary/pom.xml @@ -8,7 +8,7 @@ com.kylinolap kylin - 0.6.3 + 0.6.3-SNAPSHOT diff --git a/jdbc/pom.xml b/jdbc/pom.xml index 4fc37a7..e69cac0 100644 --- a/jdbc/pom.xml +++ b/jdbc/pom.xml @@ -11,7 +11,7 @@ com.kylinolap kylin - 0.6.3 + 0.6.3-SNAPSHOT diff --git a/job/pom.xml b/job/pom.xml index 05cb6ad..b039957 100644 --- a/job/pom.xml +++ b/job/pom.xml @@ -4,7 +4,7 @@ com.kylinolap kylin - 0.6.3 + 0.6.3-SNAPSHOT kylin-job diff --git a/metadata/pom.xml b/metadata/pom.xml index a77c88a..3e76619 100644 --- a/metadata/pom.xml +++ b/metadata/pom.xml @@ -8,7 +8,7 @@ com.kylinolap kylin - 0.6.3 + 0.6.3-SNAPSHOT diff --git a/pom.xml b/pom.xml index fc67b93..28242f8 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ com.kylinolap kylin pom - 0.6.3 + 0.6.3-SNAPSHOT Kylin:HadoopOLAPEngine @@ -493,7 +493,7 @@ scm:git:git@github.com:KylinOLAP/Kylin.git scm:git:git@github.com:KylinOLAP/Kylin.git scm:git:git@github.com:KylinOLAP/Kylin.git - kylin-0.6.3 + HEAD diff --git a/query/pom.xml b/query/pom.xml index d7ca6b0..189eb82 100644 --- a/query/pom.xml +++ b/query/pom.xml @@ -9,7 +9,7 @@ com.kylinolap kylin - 0.6.3 + 0.6.3-SNAPSHOT diff --git a/server/pom.xml b/server/pom.xml index 9315dfe..fea3397 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -9,7 +9,7 @@ com.kylinolap kylin - 0.6.3 + 0.6.3-SNAPSHOT diff --git a/storage/pom.xml b/storage/pom.xml index 520fd0e..46c43e5 100644 --- a/storage/pom.xml +++ b/storage/pom.xml @@ -7,7 +7,7 @@ com.kylinolap kylin - 0.6.3 + 0.6.3-SNAPSHOT From 5fcb6c66a01f064e9aa1c1c77cf8f68562fb82a8 Mon Sep 17 00:00:00 2001 From: Luke Han Date: Fri, 28 Nov 2014 17:03:13 +0800 Subject: [PATCH 05/35] update mail subject --- job/src/main/java/com/kylinolap/job/flow/JobFlowListener.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/job/src/main/java/com/kylinolap/job/flow/JobFlowListener.java b/job/src/main/java/com/kylinolap/job/flow/JobFlowListener.java index b4ab3ea..5c35879 100644 --- a/job/src/main/java/com/kylinolap/job/flow/JobFlowListener.java +++ b/job/src/main/java/com/kylinolap/job/flow/JobFlowListener.java @@ -398,7 +398,7 @@ protected void notifyUsers(JobInstance jobInstance, JobEngineConfig engineConfig } if (users.size() > 0) { - mailService.sendMail(users, "[Kylin Cube Build Job]-" + cubeName + "-" + finalStatus, content); + mailService.sendMail(users, "["+ finalStatus + "] - [Kylin Cube Build Job]-" + cubeName, content); } } catch (IOException e) { log.error(e.getLocalizedMessage(), e); From bf1e976d1f13ff251e75bb7d7e71e3485aa66825 Mon Sep 17 00:00:00 2001 From: shaofengshi Date: Fri, 28 Nov 2014 17:30:32 +0800 Subject: [PATCH 06/35] Move the mail related settings to kylin.properties --- common/pom.xml | 6 +- .../java/com/kylinolap/common/KylinConfig.java | 10 +++ .../com/kylinolap/common/util/MailService.java | 91 ++++++++++++++++++++++ examples/test_case_data/sandbox/kylin.properties | 11 ++- .../com/kylinolap/job/flow/JobFlowListener.java | 3 +- .../java/com/kylinolap/job/tools/MailService.java | 91 ---------------------- 6 files changed, 118 insertions(+), 94 deletions(-) create mode 100644 common/src/main/java/com/kylinolap/common/util/MailService.java delete mode 100644 job/src/main/java/com/kylinolap/job/tools/MailService.java diff --git a/common/pom.xml b/common/pom.xml index 654df7e..32a46c6 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -45,7 +45,11 @@ - + + org.apache.commons + commons-email + 1.1 + org.apache.hadoop hadoop-common diff --git a/common/src/main/java/com/kylinolap/common/KylinConfig.java b/common/src/main/java/com/kylinolap/common/KylinConfig.java index 2e834d5..34b348d 100644 --- a/common/src/main/java/com/kylinolap/common/KylinConfig.java +++ b/common/src/main/java/com/kylinolap/common/KylinConfig.java @@ -123,6 +123,16 @@ */ public static final String KYLIN_CONF_PROPERTIES_FILE = "kylin.properties"; + public static final String MAIL_ENABLED = "mail.enabled"; + + public static final String MAIL_HOST = "mail.host"; + + public static final String MAIL_USERNAME = "mail.username"; + + public static final String MAIL_PASSWORD = "mail.password"; + + public static final String MAIL_SENDER = "mail.sender"; + private static final Logger logger = LoggerFactory.getLogger(KylinConfig.class); // static cached instances diff --git a/common/src/main/java/com/kylinolap/common/util/MailService.java b/common/src/main/java/com/kylinolap/common/util/MailService.java new file mode 100644 index 0000000..5bb0ccb --- /dev/null +++ b/common/src/main/java/com/kylinolap/common/util/MailService.java @@ -0,0 +1,91 @@ +/* + * Copyright 2013-2014 eBay Software Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kylinolap.common.util; + +import java.io.IOException; +import java.util.List; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.commons.mail.Email; +import org.apache.commons.mail.EmailException; +import org.apache.commons.mail.HtmlEmail; + +import com.kylinolap.common.KylinConfig; + +/** + * @author xduo + * + */ +public class MailService { + + private Boolean enabled = Boolean.TRUE; + private String host; + private String username; + private String password; + private String sender; + + private static final Log logger = LogFactory.getLog(MailService.class); + + public MailService() { + this(KylinConfig.getInstanceFromEnv()); + } + + public MailService(KylinConfig config) { + enabled = "true".equalsIgnoreCase(config.getProperty(KylinConfig.MAIL_ENABLED, "true")); + host = config.getProperty(KylinConfig.MAIL_HOST, ""); + username = config.getProperty(KylinConfig.MAIL_USERNAME, ""); + password = config.getProperty(KylinConfig.MAIL_PASSWORD, ""); + sender = config.getProperty(KylinConfig.MAIL_SENDER, ""); + + if (enabled) { + assert !host.isEmpty(); + } + } + + public void sendMail(List receivers, String subject, String content) throws IOException { + + if (!enabled) { + logger.info("Email service is disabled; this mail will not be delivered: " + subject); + logger.info("To enable mail service, set 'mail.enabled=true' in kylin.properties"); + return; + } + + Email email = new HtmlEmail(); + email.setHostName(host); + if (username != null && username.trim().length() > 0) { + email.setAuthentication(username, password); + } + + email.setDebug(true); + try { + for (String receiver : receivers) { + email.addTo(receiver); + } + + email.setFrom(sender); + email.setSubject(subject); + email.setCharset("UTF-8"); + ((HtmlEmail) email).setHtmlMsg(content); + email.send(); + email.getMailSession(); + + } catch (EmailException e) { + logger.error(e); + } + } +} diff --git a/examples/test_case_data/sandbox/kylin.properties b/examples/test_case_data/sandbox/kylin.properties index 4cf4f07..0675b21 100644 --- a/examples/test_case_data/sandbox/kylin.properties +++ b/examples/test_case_data/sandbox/kylin.properties @@ -63,4 +63,13 @@ ldap.service.groupSearchBase= acl.adminRole= acl.defaultRole= ganglia.group= -ganglia.port=8664 \ No newline at end of file +ganglia.port=8664 + +## Config for mail service + +# If true, will send email notification; +mail.enabled=true +mail.host=atom.corp.ebay.com +mail.username=_kylin_ldap +mail.password= +mail.sender=DL-eBay-Kylin@corp.ebay.com \ No newline at end of file diff --git a/job/src/main/java/com/kylinolap/job/flow/JobFlowListener.java b/job/src/main/java/com/kylinolap/job/flow/JobFlowListener.java index 5c35879..db0be3d 100644 --- a/job/src/main/java/com/kylinolap/job/flow/JobFlowListener.java +++ b/job/src/main/java/com/kylinolap/job/flow/JobFlowListener.java @@ -25,6 +25,7 @@ import java.util.concurrent.ConcurrentHashMap; import com.kylinolap.cube.CubeSegmentStatusEnum; + import org.apache.commons.lang.exception.ExceptionUtils; import org.quartz.JobDataMap; import org.quartz.JobDetail; @@ -38,6 +39,7 @@ import org.slf4j.LoggerFactory; import com.kylinolap.common.KylinConfig; +import com.kylinolap.common.util.MailService; import com.kylinolap.cube.CubeInstance; import com.kylinolap.cube.CubeManager; import com.kylinolap.cube.CubeSegment; @@ -49,7 +51,6 @@ import com.kylinolap.job.constant.JobStatusEnum; import com.kylinolap.job.constant.JobStepStatusEnum; import com.kylinolap.job.engine.JobEngineConfig; -import com.kylinolap.job.tools.MailService; /** * Handle kylin job and cube change update. diff --git a/job/src/main/java/com/kylinolap/job/tools/MailService.java b/job/src/main/java/com/kylinolap/job/tools/MailService.java deleted file mode 100644 index 5735dbb..0000000 --- a/job/src/main/java/com/kylinolap/job/tools/MailService.java +++ /dev/null @@ -1,91 +0,0 @@ -/* - * Copyright 2013-2014 eBay Software Foundation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.kylinolap.job.tools; - -import java.io.IOException; -import java.util.List; - -import org.apache.commons.mail.Email; -import org.apache.commons.mail.EmailException; -import org.apache.commons.mail.HtmlEmail; - -/** - * @author xduo - * - */ -public class MailService { - - private String host = "atom.corp.ebay.com"; - private String username = "_kylin_ldap"; - private String password; - private String sender = "DL-eBay-Kylin@corp.ebay.com"; - - public String getHost() { - return host; - } - - public void setHost(String host) { - this.host = host; - } - - public String getUsername() { - return username; - } - - public void setUsername(String username) { - this.username = username; - } - - public String getPassword() { - return password; - } - - public void setPassword(String password) { - this.password = password; - } - - public String getSender() { - return sender; - } - - public void setSender(String sender) { - this.sender = sender; - } - - public void sendMail(List receivers, String subject, String content) throws IOException { - - Email email = new HtmlEmail(); - email.setHostName(host); - email.setDebug(true); - try { - for (String receiver : receivers) { - email.addTo(receiver); - } - - email.setFrom(sender); - email.setSubject(subject); - email.setCharset("UTF-8"); - ((HtmlEmail) email).setHtmlMsg(content); - email.send(); - email.getMailSession(); - - System.out.println("!!"); - } catch (EmailException e) { - e.printStackTrace(); - } - } -} From 8598056970f715336cc81b47c81c63764f30f65c Mon Sep 17 00:00:00 2001 From: shaofengshi Date: Fri, 28 Nov 2014 17:51:17 +0800 Subject: [PATCH 07/35] Add a test case for MailService --- .../com/kylinolap/common/util/MailService.java | 17 +++++-- .../com/kylinolap/common/util/MailServiceTest.java | 59 ++++++++++++++++++++++ 2 files changed, 73 insertions(+), 3 deletions(-) create mode 100644 common/src/test/java/com/kylinolap/common/util/MailServiceTest.java diff --git a/common/src/main/java/com/kylinolap/common/util/MailService.java b/common/src/main/java/com/kylinolap/common/util/MailService.java index 5bb0ccb..1d76ffe 100644 --- a/common/src/main/java/com/kylinolap/common/util/MailService.java +++ b/common/src/main/java/com/kylinolap/common/util/MailService.java @@ -57,12 +57,20 @@ public MailService(KylinConfig config) { } } - public void sendMail(List receivers, String subject, String content) throws IOException { + /** + * + * @param receivers + * @param subject + * @param content + * @return true or false indicating whether the email was delivered successfully + * @throws IOException + */ + public boolean sendMail(List receivers, String subject, String content) throws IOException { if (!enabled) { logger.info("Email service is disabled; this mail will not be delivered: " + subject); logger.info("To enable mail service, set 'mail.enabled=true' in kylin.properties"); - return; + return false; } Email email = new HtmlEmail(); @@ -71,7 +79,7 @@ public void sendMail(List receivers, String subject, String content) thr email.setAuthentication(username, password); } - email.setDebug(true); + //email.setDebug(true); try { for (String receiver : receivers) { email.addTo(receiver); @@ -86,6 +94,9 @@ public void sendMail(List receivers, String subject, String content) thr } catch (EmailException e) { logger.error(e); + return false; } + + return true; } } diff --git a/common/src/test/java/com/kylinolap/common/util/MailServiceTest.java b/common/src/test/java/com/kylinolap/common/util/MailServiceTest.java new file mode 100644 index 0000000..33a51e8 --- /dev/null +++ b/common/src/test/java/com/kylinolap/common/util/MailServiceTest.java @@ -0,0 +1,59 @@ +/* + * Copyright 2013-2014 eBay Software Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package com.kylinolap.common.util; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import org.junit.Test; + +import com.kylinolap.common.KylinConfig; + +public class MailServiceTest { + + @Test + public void testSendEmail() throws IOException { + + KylinConfig config = KylinConfig.getInstanceForTest(AbstractKylinTestCase.SANDBOX_TEST_DATA); + + MailService mailservice = new MailService(config); + boolean sent = sendTestEmail(mailservice); + assert sent; + + // set mail.enabled=false, and run again, this time should be no mail delviered + config.setProperty(KylinConfig.MAIL_ENABLED, "false"); + mailservice = new MailService(config); + sent = sendTestEmail(mailservice); + assert !sent; + + } + + private boolean sendTestEmail(MailService mailservice) { + + List receivers = new ArrayList(1); + receivers.add("shaoshi@ebay.com"); + try { + return mailservice.sendMail(receivers, "A test email from Kylin", "Hello!"); + } catch (IOException e) { + e.printStackTrace(); + } + + return false; + } +} From 69245ca840f49f329370fb08c267e651c38efe8f Mon Sep 17 00:00:00 2001 From: honma Date: Fri, 28 Nov 2014 18:42:26 +0800 Subject: [PATCH 08/35] fix project get table count bug --- .../com/kylinolap/cube/project/ProjectInstance.java | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/cube/src/main/java/com/kylinolap/cube/project/ProjectInstance.java b/cube/src/main/java/com/kylinolap/cube/project/ProjectInstance.java index f8939e4..1d4023e 100644 --- a/cube/src/main/java/com/kylinolap/cube/project/ProjectInstance.java +++ b/cube/src/main/java/com/kylinolap/cube/project/ProjectInstance.java @@ -42,8 +42,8 @@ @JsonProperty("tables") private Set tables; - - + + @JsonProperty("owner") private String owner; @@ -161,8 +161,8 @@ public void addCube(String cubeName) { public void setCubes(List cubes) { this.cubes = cubes; - } - + } + public void setTables(Set tables) { this.tables = tables; } @@ -178,18 +178,19 @@ public void removeTable(String tableName) { } public int getTablesCount() { - return cubes.size(); + return this.getTables().size(); } public void addTable(String tableName) { - tableName = tableName.toUpperCase(); + tableName = tableName.toUpperCase(); this.getTables().add(tableName); } //will return new Set for null public Set getTables() { - tables = tables==null?new TreeSet():tables; - return tables; } + tables = tables == null ? new TreeSet() : tables; + return tables; + } public String getOwner() { return owner; @@ -210,7 +211,7 @@ public void setLastUpdateTime(String lastUpdateTime) { public void recordUpdateTime(long timeMillis) { this.lastUpdateTime = formatTime(timeMillis); } - + public void init() { if (name == null) From c78850792ca48cce0f1381abfc3d19204c99ff05 Mon Sep 17 00:00:00 2001 From: jiazhong Date: Fri, 28 Nov 2014 19:11:23 +0800 Subject: [PATCH 09/35] print exception log and new reload logic on cube&job js --- .../main/java/com/kylinolap/rest/controller/CubeController.java | 1 + webapp/app/js/controllers/cubes.js | 7 ++++--- webapp/app/js/controllers/job.js | 2 +- webapp/app/js/directives/directives.js | 1 + 4 files changed, 7 insertions(+), 4 deletions(-) diff --git a/server/src/main/java/com/kylinolap/rest/controller/CubeController.java b/server/src/main/java/com/kylinolap/rest/controller/CubeController.java index 5f6f214..7a36d77 100644 --- a/server/src/main/java/com/kylinolap/rest/controller/CubeController.java +++ b/server/src/main/java/com/kylinolap/rest/controller/CubeController.java @@ -357,6 +357,7 @@ public CubeRequest updateCubeDesc(@RequestBody CubeRequest cubeRequest) { throw new ForbiddenException("You don't have right to update this cube."); } catch (Exception e) { logger.error("Failed to deal with the request.", e); + e.printStackTrace(); throw new InternalErrorException("Failed to deal with the request.", e); } diff --git a/webapp/app/js/controllers/cubes.js b/webapp/app/js/controllers/cubes.js index edffc56..d48c103 100644 --- a/webapp/app/js/controllers/cubes.js +++ b/webapp/app/js/controllers/cubes.js @@ -73,9 +73,10 @@ KylinApp }; $scope.$watch('project.selectedProject', function (newValue, oldValue) { - if(newValue){ - $scope.cubes=[]; - $scope.reload(); + //exclude when refresh page oldValue=null,first time set value for project (will have page auto reload ,incase duplicate) oldvalue is null + if(oldValue){ + $scope.cubes=[]; + $scope.reload(); } }); diff --git a/webapp/app/js/controllers/job.js b/webapp/app/js/controllers/job.js index 84afd1e..68b1f37 100644 --- a/webapp/app/js/controllers/job.js +++ b/webapp/app/js/controllers/job.js @@ -82,7 +82,7 @@ KylinApp $scope.$watch('project.selectedProject', function (newValue, oldValue) { - if(newValue){ + if(oldValue){ $scope.jobs={}; $scope.state.projectName = newValue; $scope.reload(); diff --git a/webapp/app/js/directives/directives.js b/webapp/app/js/directives/directives.js index d024fcd..264cb05 100644 --- a/webapp/app/js/directives/directives.js +++ b/webapp/app/js/directives/directives.js @@ -16,6 +16,7 @@ KylinApp.directive('kylinPagination', function ($parse, $q) { scope.loadFunc = $parse(attrs.loadFunc)(scope.$parent); scope.autoLoad = true; + scope.$watch("action.reload", function (newValue, oldValue) { if (newValue != oldValue) { scope.reload(); From f3975bec90712594cfff8a458aad97e0371a790a Mon Sep 17 00:00:00 2001 From: "Li, Yang" Date: Fri, 28 Nov 2014 20:21:42 +0800 Subject: [PATCH 10/35] debug for reproduce a strange issue --- query/src/main/java/com/kylinolap/query/relnode/OLAPAggregateRel.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/query/src/main/java/com/kylinolap/query/relnode/OLAPAggregateRel.java b/query/src/main/java/com/kylinolap/query/relnode/OLAPAggregateRel.java index 33a403d..b6cfaa9 100644 --- a/query/src/main/java/com/kylinolap/query/relnode/OLAPAggregateRel.java +++ b/query/src/main/java/com/kylinolap/query/relnode/OLAPAggregateRel.java @@ -181,6 +181,10 @@ private TblColRef buildRewriteColumn(FunctionDesc aggFunc) { if (aggFunc.needRewrite()) { ColumnDesc column = new ColumnDesc(); column.setName(aggFunc.getRewriteFieldName()); + logger.info(""+this.context); + logger.info(""+this.context.firstTableScan); + logger.info(""+this.context.firstTableScan.getOlapTable()); + logger.info(""+this.context.firstTableScan.getOlapTable().getSourceTable()); TableDesc table = this.context.firstTableScan.getOlapTable().getSourceTable(); column.setTable(table); colRef = new TblColRef(column); From a2273d8eeb244f2177221be4c8a04797d7eeef0b Mon Sep 17 00:00:00 2001 From: "Li, Yang" Date: Fri, 28 Nov 2014 20:37:55 +0800 Subject: [PATCH 11/35] enable `explain plan for` from GUI --- .../src/main/java/com/kylinolap/query/relnode/OLAPContext.java | 3 ++- .../src/main/java/com/kylinolap/rest/service/QueryService.java | 10 ++++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/query/src/main/java/com/kylinolap/query/relnode/OLAPContext.java b/query/src/main/java/com/kylinolap/query/relnode/OLAPContext.java index 83dd18a..7de1817 100644 --- a/query/src/main/java/com/kylinolap/query/relnode/OLAPContext.java +++ b/query/src/main/java/com/kylinolap/query/relnode/OLAPContext.java @@ -64,7 +64,8 @@ public static void registerContext(OLAPContext ctx) { } public static Collection getThreadLocalContexts() { - return _localContexts.get().values(); + Map map = _localContexts.get(); + return map == null ? null : map.values(); } public static OLAPContext getThreadLocalContextById(int id) { diff --git a/server/src/main/java/com/kylinolap/rest/service/QueryService.java b/server/src/main/java/com/kylinolap/rest/service/QueryService.java index 5e90e85..9277a08 100644 --- a/server/src/main/java/com/kylinolap/rest/service/QueryService.java +++ b/server/src/main/java/com/kylinolap/rest/service/QueryService.java @@ -384,10 +384,12 @@ private SQLResponse execute(String sql, SQLRequest sqlRequest) throws Exception boolean isPartialResult = false; String cube = ""; long totalScanCount = 0; - for (OLAPContext ctx : OLAPContext.getThreadLocalContexts()) { - isPartialResult |= ctx.storageContext.isPartialResultReturned(); - cube = ctx.cubeInstance.getName(); - totalScanCount += ctx.storageContext.getTotalScanCount(); + if (OLAPContext.getThreadLocalContexts() != null) { // contexts can be null in case of 'explain plan for' + for (OLAPContext ctx : OLAPContext.getThreadLocalContexts()) { + isPartialResult |= ctx.storageContext.isPartialResultReturned(); + cube = ctx.cubeInstance.getName(); + totalScanCount += ctx.storageContext.getTotalScanCount(); + } } SQLResponse response = new SQLResponse(columnMetas, results, cube, 0, false, null, isPartialResult); From 20a8ccf9422820f0c32a65da87657954a22c6250 Mon Sep 17 00:00:00 2001 From: "Li, Yang" Date: Fri, 28 Nov 2014 20:50:50 +0800 Subject: [PATCH 12/35] drop debug info --- query/src/main/java/com/kylinolap/query/relnode/OLAPAggregateRel.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/query/src/main/java/com/kylinolap/query/relnode/OLAPAggregateRel.java b/query/src/main/java/com/kylinolap/query/relnode/OLAPAggregateRel.java index b6cfaa9..33a403d 100644 --- a/query/src/main/java/com/kylinolap/query/relnode/OLAPAggregateRel.java +++ b/query/src/main/java/com/kylinolap/query/relnode/OLAPAggregateRel.java @@ -181,10 +181,6 @@ private TblColRef buildRewriteColumn(FunctionDesc aggFunc) { if (aggFunc.needRewrite()) { ColumnDesc column = new ColumnDesc(); column.setName(aggFunc.getRewriteFieldName()); - logger.info(""+this.context); - logger.info(""+this.context.firstTableScan); - logger.info(""+this.context.firstTableScan.getOlapTable()); - logger.info(""+this.context.firstTableScan.getOlapTable().getSourceTable()); TableDesc table = this.context.firstTableScan.getOlapTable().getSourceTable(); column.setTable(table); colRef = new TblColRef(column); From 4e51bc4bd3435257481d2c370509ccb23b4c94e0 Mon Sep 17 00:00:00 2001 From: "Li, Yang" Date: Fri, 28 Nov 2014 21:07:35 +0800 Subject: [PATCH 13/35] create new project 'atopcalcite' to ensure override of calcite goes before calcite*.jar --- .../.settings/org.eclipse.core.resources.prefs | 6 + atopcalcite/.settings/org.eclipse.jdt.core.prefs | 379 ++ atopcalcite/.settings/org.eclipse.jdt.ui.prefs | 7 + atopcalcite/pom.xml | 28 + .../net/hydromatic/optiq/runtime/SqlFunctions.java | 1705 +++++++ .../org/eigenbase/sql2rel/SqlToRelConverter.java | 4782 ++++++++++++++++++++ pom.xml | 1 + query/pom.xml | 11 +- .../net/hydromatic/optiq/runtime/SqlFunctions.java | 1705 ------- .../org/eigenbase/sql2rel/SqlToRelConverter.java | 4782 -------------------- 10 files changed, 6912 insertions(+), 6494 deletions(-) create mode 100644 atopcalcite/.settings/org.eclipse.core.resources.prefs create mode 100644 atopcalcite/.settings/org.eclipse.jdt.core.prefs create mode 100644 atopcalcite/.settings/org.eclipse.jdt.ui.prefs create mode 100644 atopcalcite/pom.xml create mode 100644 atopcalcite/src/main/java/net/hydromatic/optiq/runtime/SqlFunctions.java create mode 100644 atopcalcite/src/main/java/org/eigenbase/sql2rel/SqlToRelConverter.java delete mode 100644 query/src/main/java/net/hydromatic/optiq/runtime/SqlFunctions.java delete mode 100644 query/src/main/java/org/eigenbase/sql2rel/SqlToRelConverter.java diff --git a/atopcalcite/.settings/org.eclipse.core.resources.prefs b/atopcalcite/.settings/org.eclipse.core.resources.prefs new file mode 100644 index 0000000..29abf99 --- /dev/null +++ b/atopcalcite/.settings/org.eclipse.core.resources.prefs @@ -0,0 +1,6 @@ +eclipse.preferences.version=1 +encoding//src/main/java=UTF-8 +encoding//src/main/resources=UTF-8 +encoding//src/test/java=UTF-8 +encoding//src/test/resources=UTF-8 +encoding/=UTF-8 diff --git a/atopcalcite/.settings/org.eclipse.jdt.core.prefs b/atopcalcite/.settings/org.eclipse.jdt.core.prefs new file mode 100644 index 0000000..7c91e8d --- /dev/null +++ b/atopcalcite/.settings/org.eclipse.jdt.core.prefs @@ -0,0 +1,379 @@ +eclipse.preferences.version=1 +org.eclipse.jdt.core.compiler.annotation.inheritNullAnnotations=disabled +org.eclipse.jdt.core.compiler.annotation.missingNonNullByDefaultAnnotation=ignore +org.eclipse.jdt.core.compiler.annotation.nonnull=org.eclipse.jdt.annotation.NonNull +org.eclipse.jdt.core.compiler.annotation.nonnullbydefault=org.eclipse.jdt.annotation.NonNullByDefault +org.eclipse.jdt.core.compiler.annotation.nullable=org.eclipse.jdt.annotation.Nullable +org.eclipse.jdt.core.compiler.annotation.nullanalysis=disabled +org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled +org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.7 +org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve +org.eclipse.jdt.core.compiler.compliance=1.7 +org.eclipse.jdt.core.compiler.debug.lineNumber=generate +org.eclipse.jdt.core.compiler.debug.localVariable=generate +org.eclipse.jdt.core.compiler.debug.sourceFile=generate +org.eclipse.jdt.core.compiler.problem.annotationSuperInterface=warning +org.eclipse.jdt.core.compiler.problem.assertIdentifier=error +org.eclipse.jdt.core.compiler.problem.autoboxing=ignore +org.eclipse.jdt.core.compiler.problem.comparingIdentical=warning +org.eclipse.jdt.core.compiler.problem.deadCode=warning +org.eclipse.jdt.core.compiler.problem.deprecation=warning +org.eclipse.jdt.core.compiler.problem.deprecationInDeprecatedCode=disabled +org.eclipse.jdt.core.compiler.problem.deprecationWhenOverridingDeprecatedMethod=disabled +org.eclipse.jdt.core.compiler.problem.discouragedReference=warning +org.eclipse.jdt.core.compiler.problem.emptyStatement=ignore +org.eclipse.jdt.core.compiler.problem.enumIdentifier=error +org.eclipse.jdt.core.compiler.problem.explicitlyClosedAutoCloseable=ignore +org.eclipse.jdt.core.compiler.problem.fallthroughCase=ignore +org.eclipse.jdt.core.compiler.problem.fatalOptionalError=disabled +org.eclipse.jdt.core.compiler.problem.fieldHiding=ignore +org.eclipse.jdt.core.compiler.problem.finalParameterBound=warning +org.eclipse.jdt.core.compiler.problem.finallyBlockNotCompletingNormally=warning +org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning +org.eclipse.jdt.core.compiler.problem.hiddenCatchBlock=warning +org.eclipse.jdt.core.compiler.problem.includeNullInfoFromAsserts=disabled +org.eclipse.jdt.core.compiler.problem.incompatibleNonInheritedInterfaceMethod=warning +org.eclipse.jdt.core.compiler.problem.incompleteEnumSwitch=warning +org.eclipse.jdt.core.compiler.problem.indirectStaticAccess=ignore +org.eclipse.jdt.core.compiler.problem.localVariableHiding=ignore +org.eclipse.jdt.core.compiler.problem.methodWithConstructorName=warning +org.eclipse.jdt.core.compiler.problem.missingDefaultCase=ignore +org.eclipse.jdt.core.compiler.problem.missingDeprecatedAnnotation=ignore +org.eclipse.jdt.core.compiler.problem.missingEnumCaseDespiteDefault=disabled +org.eclipse.jdt.core.compiler.problem.missingHashCodeMethod=ignore +org.eclipse.jdt.core.compiler.problem.missingOverrideAnnotation=ignore +org.eclipse.jdt.core.compiler.problem.missingOverrideAnnotationForInterfaceMethodImplementation=enabled +org.eclipse.jdt.core.compiler.problem.missingSerialVersion=warning +org.eclipse.jdt.core.compiler.problem.missingSynchronizedOnInheritedMethod=ignore +org.eclipse.jdt.core.compiler.problem.noEffectAssignment=warning +org.eclipse.jdt.core.compiler.problem.noImplicitStringConversion=warning +org.eclipse.jdt.core.compiler.problem.nonExternalizedStringLiteral=ignore +org.eclipse.jdt.core.compiler.problem.nonnullParameterAnnotationDropped=warning +org.eclipse.jdt.core.compiler.problem.nullAnnotationInferenceConflict=error +org.eclipse.jdt.core.compiler.problem.nullReference=warning +org.eclipse.jdt.core.compiler.problem.nullSpecViolation=error +org.eclipse.jdt.core.compiler.problem.nullUncheckedConversion=warning +org.eclipse.jdt.core.compiler.problem.overridingPackageDefaultMethod=warning +org.eclipse.jdt.core.compiler.problem.parameterAssignment=ignore +org.eclipse.jdt.core.compiler.problem.possibleAccidentalBooleanAssignment=ignore +org.eclipse.jdt.core.compiler.problem.potentialNullReference=ignore +org.eclipse.jdt.core.compiler.problem.potentiallyUnclosedCloseable=ignore +org.eclipse.jdt.core.compiler.problem.rawTypeReference=warning +org.eclipse.jdt.core.compiler.problem.redundantNullAnnotation=warning +org.eclipse.jdt.core.compiler.problem.redundantNullCheck=ignore +org.eclipse.jdt.core.compiler.problem.redundantSpecificationOfTypeArguments=ignore +org.eclipse.jdt.core.compiler.problem.redundantSuperinterface=ignore +org.eclipse.jdt.core.compiler.problem.reportMethodCanBePotentiallyStatic=ignore +org.eclipse.jdt.core.compiler.problem.reportMethodCanBeStatic=ignore +org.eclipse.jdt.core.compiler.problem.specialParameterHidingField=disabled +org.eclipse.jdt.core.compiler.problem.staticAccessReceiver=warning +org.eclipse.jdt.core.compiler.problem.suppressOptionalErrors=disabled +org.eclipse.jdt.core.compiler.problem.suppressWarnings=enabled +org.eclipse.jdt.core.compiler.problem.syntacticNullAnalysisForFields=disabled +org.eclipse.jdt.core.compiler.problem.syntheticAccessEmulation=ignore +org.eclipse.jdt.core.compiler.problem.typeParameterHiding=warning +org.eclipse.jdt.core.compiler.problem.unavoidableGenericTypeProblems=enabled +org.eclipse.jdt.core.compiler.problem.uncheckedTypeOperation=warning +org.eclipse.jdt.core.compiler.problem.unclosedCloseable=warning +org.eclipse.jdt.core.compiler.problem.undocumentedEmptyBlock=ignore +org.eclipse.jdt.core.compiler.problem.unhandledWarningToken=warning +org.eclipse.jdt.core.compiler.problem.unnecessaryElse=ignore +org.eclipse.jdt.core.compiler.problem.unnecessaryTypeCheck=ignore +org.eclipse.jdt.core.compiler.problem.unqualifiedFieldAccess=ignore +org.eclipse.jdt.core.compiler.problem.unusedDeclaredThrownException=ignore +org.eclipse.jdt.core.compiler.problem.unusedDeclaredThrownExceptionExemptExceptionAndThrowable=enabled +org.eclipse.jdt.core.compiler.problem.unusedDeclaredThrownExceptionIncludeDocCommentReference=enabled +org.eclipse.jdt.core.compiler.problem.unusedDeclaredThrownExceptionWhenOverriding=disabled +org.eclipse.jdt.core.compiler.problem.unusedImport=warning +org.eclipse.jdt.core.compiler.problem.unusedLabel=warning +org.eclipse.jdt.core.compiler.problem.unusedLocal=warning +org.eclipse.jdt.core.compiler.problem.unusedObjectAllocation=ignore +org.eclipse.jdt.core.compiler.problem.unusedParameter=ignore +org.eclipse.jdt.core.compiler.problem.unusedParameterIncludeDocCommentReference=enabled +org.eclipse.jdt.core.compiler.problem.unusedParameterWhenImplementingAbstract=disabled +org.eclipse.jdt.core.compiler.problem.unusedParameterWhenOverridingConcrete=disabled +org.eclipse.jdt.core.compiler.problem.unusedPrivateMember=warning +org.eclipse.jdt.core.compiler.problem.unusedTypeParameter=ignore +org.eclipse.jdt.core.compiler.problem.unusedWarningToken=warning +org.eclipse.jdt.core.compiler.problem.varargsArgumentNeedCast=warning +org.eclipse.jdt.core.compiler.source=1.7 +org.eclipse.jdt.core.formatter.align_type_members_on_columns=false +org.eclipse.jdt.core.formatter.alignment_for_arguments_in_allocation_expression=16 +org.eclipse.jdt.core.formatter.alignment_for_arguments_in_annotation=0 +org.eclipse.jdt.core.formatter.alignment_for_arguments_in_enum_constant=16 +org.eclipse.jdt.core.formatter.alignment_for_arguments_in_explicit_constructor_call=16 +org.eclipse.jdt.core.formatter.alignment_for_arguments_in_method_invocation=16 +org.eclipse.jdt.core.formatter.alignment_for_arguments_in_qualified_allocation_expression=16 +org.eclipse.jdt.core.formatter.alignment_for_assignment=0 +org.eclipse.jdt.core.formatter.alignment_for_binary_expression=16 +org.eclipse.jdt.core.formatter.alignment_for_compact_if=16 +org.eclipse.jdt.core.formatter.alignment_for_conditional_expression=80 +org.eclipse.jdt.core.formatter.alignment_for_enum_constants=0 +org.eclipse.jdt.core.formatter.alignment_for_expressions_in_array_initializer=16 +org.eclipse.jdt.core.formatter.alignment_for_method_declaration=0 +org.eclipse.jdt.core.formatter.alignment_for_multiple_fields=16 +org.eclipse.jdt.core.formatter.alignment_for_parameters_in_constructor_declaration=16 +org.eclipse.jdt.core.formatter.alignment_for_parameters_in_method_declaration=16 +org.eclipse.jdt.core.formatter.alignment_for_resources_in_try=80 +org.eclipse.jdt.core.formatter.alignment_for_selector_in_method_invocation=16 +org.eclipse.jdt.core.formatter.alignment_for_superclass_in_type_declaration=16 +org.eclipse.jdt.core.formatter.alignment_for_superinterfaces_in_enum_declaration=16 +org.eclipse.jdt.core.formatter.alignment_for_superinterfaces_in_type_declaration=16 +org.eclipse.jdt.core.formatter.alignment_for_throws_clause_in_constructor_declaration=16 +org.eclipse.jdt.core.formatter.alignment_for_throws_clause_in_method_declaration=16 +org.eclipse.jdt.core.formatter.alignment_for_union_type_in_multicatch=16 +org.eclipse.jdt.core.formatter.blank_lines_after_imports=1 +org.eclipse.jdt.core.formatter.blank_lines_after_package=1 +org.eclipse.jdt.core.formatter.blank_lines_before_field=0 +org.eclipse.jdt.core.formatter.blank_lines_before_first_class_body_declaration=0 +org.eclipse.jdt.core.formatter.blank_lines_before_imports=1 +org.eclipse.jdt.core.formatter.blank_lines_before_member_type=1 +org.eclipse.jdt.core.formatter.blank_lines_before_method=1 +org.eclipse.jdt.core.formatter.blank_lines_before_new_chunk=1 +org.eclipse.jdt.core.formatter.blank_lines_before_package=0 +org.eclipse.jdt.core.formatter.blank_lines_between_import_groups=1 +org.eclipse.jdt.core.formatter.blank_lines_between_type_declarations=1 +org.eclipse.jdt.core.formatter.brace_position_for_annotation_type_declaration=end_of_line +org.eclipse.jdt.core.formatter.brace_position_for_anonymous_type_declaration=end_of_line +org.eclipse.jdt.core.formatter.brace_position_for_array_initializer=end_of_line +org.eclipse.jdt.core.formatter.brace_position_for_block=end_of_line +org.eclipse.jdt.core.formatter.brace_position_for_block_in_case=end_of_line +org.eclipse.jdt.core.formatter.brace_position_for_constructor_declaration=end_of_line +org.eclipse.jdt.core.formatter.brace_position_for_enum_constant=end_of_line +org.eclipse.jdt.core.formatter.brace_position_for_enum_declaration=end_of_line +org.eclipse.jdt.core.formatter.brace_position_for_method_declaration=end_of_line +org.eclipse.jdt.core.formatter.brace_position_for_switch=end_of_line +org.eclipse.jdt.core.formatter.brace_position_for_type_declaration=end_of_line +org.eclipse.jdt.core.formatter.comment.clear_blank_lines_in_block_comment=false +org.eclipse.jdt.core.formatter.comment.clear_blank_lines_in_javadoc_comment=false +org.eclipse.jdt.core.formatter.comment.format_block_comments=false +org.eclipse.jdt.core.formatter.comment.format_header=false +org.eclipse.jdt.core.formatter.comment.format_html=true +org.eclipse.jdt.core.formatter.comment.format_javadoc_comments=false +org.eclipse.jdt.core.formatter.comment.format_line_comments=false +org.eclipse.jdt.core.formatter.comment.format_source_code=true +org.eclipse.jdt.core.formatter.comment.indent_parameter_description=true +org.eclipse.jdt.core.formatter.comment.indent_root_tags=true +org.eclipse.jdt.core.formatter.comment.insert_new_line_before_root_tags=insert +org.eclipse.jdt.core.formatter.comment.insert_new_line_for_parameter=insert +org.eclipse.jdt.core.formatter.comment.line_length=80 +org.eclipse.jdt.core.formatter.comment.new_lines_at_block_boundaries=true +org.eclipse.jdt.core.formatter.comment.new_lines_at_javadoc_boundaries=true +org.eclipse.jdt.core.formatter.comment.preserve_white_space_between_code_and_line_comments=false +org.eclipse.jdt.core.formatter.compact_else_if=true +org.eclipse.jdt.core.formatter.continuation_indentation=2 +org.eclipse.jdt.core.formatter.continuation_indentation_for_array_initializer=2 +org.eclipse.jdt.core.formatter.disabling_tag=@formatter\:off +org.eclipse.jdt.core.formatter.enabling_tag=@formatter\:on +org.eclipse.jdt.core.formatter.format_guardian_clause_on_one_line=false +org.eclipse.jdt.core.formatter.format_line_comment_starting_on_first_column=true +org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_annotation_declaration_header=true +org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_enum_constant_header=true +org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_enum_declaration_header=true +org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_type_header=true +org.eclipse.jdt.core.formatter.indent_breaks_compare_to_cases=true +org.eclipse.jdt.core.formatter.indent_empty_lines=false +org.eclipse.jdt.core.formatter.indent_statements_compare_to_block=true +org.eclipse.jdt.core.formatter.indent_statements_compare_to_body=true +org.eclipse.jdt.core.formatter.indent_switchstatements_compare_to_cases=true +org.eclipse.jdt.core.formatter.indent_switchstatements_compare_to_switch=false +org.eclipse.jdt.core.formatter.indentation.size=4 +org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_field=insert +org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_local_variable=insert +org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_method=insert +org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_package=insert +org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_parameter=do not insert +org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_type=insert +org.eclipse.jdt.core.formatter.insert_new_line_after_label=do not insert +org.eclipse.jdt.core.formatter.insert_new_line_after_opening_brace_in_array_initializer=do not insert +org.eclipse.jdt.core.formatter.insert_new_line_at_end_of_file_if_missing=do not insert +org.eclipse.jdt.core.formatter.insert_new_line_before_catch_in_try_statement=do not insert +org.eclipse.jdt.core.formatter.insert_new_line_before_closing_brace_in_array_initializer=do not insert +org.eclipse.jdt.core.formatter.insert_new_line_before_else_in_if_statement=do not insert +org.eclipse.jdt.core.formatter.insert_new_line_before_finally_in_try_statement=do not insert +org.eclipse.jdt.core.formatter.insert_new_line_before_while_in_do_statement=do not insert +org.eclipse.jdt.core.formatter.insert_new_line_in_empty_annotation_declaration=insert +org.eclipse.jdt.core.formatter.insert_new_line_in_empty_anonymous_type_declaration=insert +org.eclipse.jdt.core.formatter.insert_new_line_in_empty_block=insert +org.eclipse.jdt.core.formatter.insert_new_line_in_empty_enum_constant=insert +org.eclipse.jdt.core.formatter.insert_new_line_in_empty_enum_declaration=insert +org.eclipse.jdt.core.formatter.insert_new_line_in_empty_method_body=insert +org.eclipse.jdt.core.formatter.insert_new_line_in_empty_type_declaration=insert +org.eclipse.jdt.core.formatter.insert_space_after_and_in_type_parameter=insert +org.eclipse.jdt.core.formatter.insert_space_after_assignment_operator=insert +org.eclipse.jdt.core.formatter.insert_space_after_at_in_annotation=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_at_in_annotation_type_declaration=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_binary_operator=insert +org.eclipse.jdt.core.formatter.insert_space_after_closing_angle_bracket_in_type_arguments=insert +org.eclipse.jdt.core.formatter.insert_space_after_closing_angle_bracket_in_type_parameters=insert +org.eclipse.jdt.core.formatter.insert_space_after_closing_brace_in_block=insert +org.eclipse.jdt.core.formatter.insert_space_after_closing_paren_in_cast=insert +org.eclipse.jdt.core.formatter.insert_space_after_colon_in_assert=insert +org.eclipse.jdt.core.formatter.insert_space_after_colon_in_case=insert +org.eclipse.jdt.core.formatter.insert_space_after_colon_in_conditional=insert +org.eclipse.jdt.core.formatter.insert_space_after_colon_in_for=insert +org.eclipse.jdt.core.formatter.insert_space_after_colon_in_labeled_statement=insert +org.eclipse.jdt.core.formatter.insert_space_after_comma_in_allocation_expression=insert +org.eclipse.jdt.core.formatter.insert_space_after_comma_in_annotation=insert +org.eclipse.jdt.core.formatter.insert_space_after_comma_in_array_initializer=insert +org.eclipse.jdt.core.formatter.insert_space_after_comma_in_constructor_declaration_parameters=insert +org.eclipse.jdt.core.formatter.insert_space_after_comma_in_constructor_declaration_throws=insert +org.eclipse.jdt.core.formatter.insert_space_after_comma_in_enum_constant_arguments=insert +org.eclipse.jdt.core.formatter.insert_space_after_comma_in_enum_declarations=insert +org.eclipse.jdt.core.formatter.insert_space_after_comma_in_explicitconstructorcall_arguments=insert +org.eclipse.jdt.core.formatter.insert_space_after_comma_in_for_increments=insert +org.eclipse.jdt.core.formatter.insert_space_after_comma_in_for_inits=insert +org.eclipse.jdt.core.formatter.insert_space_after_comma_in_method_declaration_parameters=insert +org.eclipse.jdt.core.formatter.insert_space_after_comma_in_method_declaration_throws=insert +org.eclipse.jdt.core.formatter.insert_space_after_comma_in_method_invocation_arguments=insert +org.eclipse.jdt.core.formatter.insert_space_after_comma_in_multiple_field_declarations=insert +org.eclipse.jdt.core.formatter.insert_space_after_comma_in_multiple_local_declarations=insert +org.eclipse.jdt.core.formatter.insert_space_after_comma_in_parameterized_type_reference=insert +org.eclipse.jdt.core.formatter.insert_space_after_comma_in_superinterfaces=insert +org.eclipse.jdt.core.formatter.insert_space_after_comma_in_type_arguments=insert +org.eclipse.jdt.core.formatter.insert_space_after_comma_in_type_parameters=insert +org.eclipse.jdt.core.formatter.insert_space_after_ellipsis=insert +org.eclipse.jdt.core.formatter.insert_space_after_opening_angle_bracket_in_parameterized_type_reference=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_opening_angle_bracket_in_type_arguments=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_opening_angle_bracket_in_type_parameters=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_opening_brace_in_array_initializer=insert +org.eclipse.jdt.core.formatter.insert_space_after_opening_bracket_in_array_allocation_expression=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_opening_bracket_in_array_reference=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_annotation=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_cast=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_catch=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_constructor_declaration=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_enum_constant=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_for=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_if=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_method_declaration=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_method_invocation=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_parenthesized_expression=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_switch=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_synchronized=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_try=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_while=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_postfix_operator=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_prefix_operator=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_question_in_conditional=insert +org.eclipse.jdt.core.formatter.insert_space_after_question_in_wildcard=do not insert +org.eclipse.jdt.core.formatter.insert_space_after_semicolon_in_for=insert +org.eclipse.jdt.core.formatter.insert_space_after_semicolon_in_try_resources=insert +org.eclipse.jdt.core.formatter.insert_space_after_unary_operator=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_and_in_type_parameter=insert +org.eclipse.jdt.core.formatter.insert_space_before_assignment_operator=insert +org.eclipse.jdt.core.formatter.insert_space_before_at_in_annotation_type_declaration=insert +org.eclipse.jdt.core.formatter.insert_space_before_binary_operator=insert +org.eclipse.jdt.core.formatter.insert_space_before_closing_angle_bracket_in_parameterized_type_reference=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_closing_angle_bracket_in_type_arguments=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_closing_angle_bracket_in_type_parameters=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_closing_brace_in_array_initializer=insert +org.eclipse.jdt.core.formatter.insert_space_before_closing_bracket_in_array_allocation_expression=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_closing_bracket_in_array_reference=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_annotation=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_cast=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_catch=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_constructor_declaration=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_enum_constant=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_for=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_if=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_method_declaration=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_method_invocation=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_parenthesized_expression=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_switch=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_synchronized=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_try=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_while=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_colon_in_assert=insert +org.eclipse.jdt.core.formatter.insert_space_before_colon_in_case=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_colon_in_conditional=insert +org.eclipse.jdt.core.formatter.insert_space_before_colon_in_default=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_colon_in_for=insert +org.eclipse.jdt.core.formatter.insert_space_before_colon_in_labeled_statement=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_comma_in_allocation_expression=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_comma_in_annotation=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_comma_in_array_initializer=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_comma_in_constructor_declaration_parameters=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_comma_in_constructor_declaration_throws=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_comma_in_enum_constant_arguments=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_comma_in_enum_declarations=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_comma_in_explicitconstructorcall_arguments=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_comma_in_for_increments=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_comma_in_for_inits=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_comma_in_method_declaration_parameters=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_comma_in_method_declaration_throws=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_comma_in_method_invocation_arguments=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_comma_in_multiple_field_declarations=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_comma_in_multiple_local_declarations=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_comma_in_parameterized_type_reference=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_comma_in_superinterfaces=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_comma_in_type_arguments=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_comma_in_type_parameters=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_ellipsis=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_angle_bracket_in_parameterized_type_reference=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_angle_bracket_in_type_arguments=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_angle_bracket_in_type_parameters=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_annotation_type_declaration=insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_anonymous_type_declaration=insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_array_initializer=insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_block=insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_constructor_declaration=insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_enum_constant=insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_enum_declaration=insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_method_declaration=insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_switch=insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_type_declaration=insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_bracket_in_array_allocation_expression=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_bracket_in_array_reference=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_bracket_in_array_type_reference=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_annotation=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_annotation_type_member_declaration=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_catch=insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_constructor_declaration=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_enum_constant=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_for=insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_if=insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_method_declaration=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_method_invocation=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_parenthesized_expression=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_switch=insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_synchronized=insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_try=insert +org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_while=insert +org.eclipse.jdt.core.formatter.insert_space_before_parenthesized_expression_in_return=insert +org.eclipse.jdt.core.formatter.insert_space_before_parenthesized_expression_in_throw=insert +org.eclipse.jdt.core.formatter.insert_space_before_postfix_operator=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_prefix_operator=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_question_in_conditional=insert +org.eclipse.jdt.core.formatter.insert_space_before_question_in_wildcard=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_semicolon=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_semicolon_in_for=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_semicolon_in_try_resources=do not insert +org.eclipse.jdt.core.formatter.insert_space_before_unary_operator=do not insert +org.eclipse.jdt.core.formatter.insert_space_between_brackets_in_array_type_reference=do not insert +org.eclipse.jdt.core.formatter.insert_space_between_empty_braces_in_array_initializer=do not insert +org.eclipse.jdt.core.formatter.insert_space_between_empty_brackets_in_array_allocation_expression=do not insert +org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_annotation_type_member_declaration=do not insert +org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_constructor_declaration=do not insert +org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_enum_constant=do not insert +org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_method_declaration=do not insert +org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_method_invocation=do not insert +org.eclipse.jdt.core.formatter.join_lines_in_comments=true +org.eclipse.jdt.core.formatter.join_wrapped_lines=true +org.eclipse.jdt.core.formatter.keep_else_statement_on_same_line=false +org.eclipse.jdt.core.formatter.keep_empty_array_initializer_on_one_line=false +org.eclipse.jdt.core.formatter.keep_imple_if_on_one_line=false +org.eclipse.jdt.core.formatter.keep_then_statement_on_same_line=false +org.eclipse.jdt.core.formatter.lineSplit=999 +org.eclipse.jdt.core.formatter.never_indent_block_comments_on_first_column=false +org.eclipse.jdt.core.formatter.never_indent_line_comments_on_first_column=false +org.eclipse.jdt.core.formatter.number_of_blank_lines_at_beginning_of_method_body=0 +org.eclipse.jdt.core.formatter.number_of_empty_lines_to_preserve=1 +org.eclipse.jdt.core.formatter.put_empty_statement_on_new_line=true +org.eclipse.jdt.core.formatter.tabulation.char=space +org.eclipse.jdt.core.formatter.tabulation.size=4 +org.eclipse.jdt.core.formatter.use_on_off_tags=false +org.eclipse.jdt.core.formatter.use_tabs_only_for_leading_indentations=false +org.eclipse.jdt.core.formatter.wrap_before_binary_operator=true +org.eclipse.jdt.core.formatter.wrap_before_or_operator_multicatch=true +org.eclipse.jdt.core.formatter.wrap_outer_expressions_when_nested=true diff --git a/atopcalcite/.settings/org.eclipse.jdt.ui.prefs b/atopcalcite/.settings/org.eclipse.jdt.ui.prefs new file mode 100644 index 0000000..681b0b0 --- /dev/null +++ b/atopcalcite/.settings/org.eclipse.jdt.ui.prefs @@ -0,0 +1,7 @@ +eclipse.preferences.version=1 +formatter_profile=_Space Indent & Long Lines +formatter_settings_version=12 +org.eclipse.jdt.ui.ignorelowercasenames=true +org.eclipse.jdt.ui.importorder=java;javax;org;com; +org.eclipse.jdt.ui.ondemandthreshold=99 +org.eclipse.jdt.ui.staticondemandthreshold=1 diff --git a/atopcalcite/pom.xml b/atopcalcite/pom.xml new file mode 100644 index 0000000..e36f434 --- /dev/null +++ b/atopcalcite/pom.xml @@ -0,0 +1,28 @@ + + 4.0.0 + + atopcalcite + jar + Kylin:AtopCalcite + + + com.kylinolap + kylin + 0.6.3-SNAPSHOT + + + + + + + + org.apache.calcite + calcite-core + + + org.apache.calcite + calcite-avatica + + + + diff --git a/atopcalcite/src/main/java/net/hydromatic/optiq/runtime/SqlFunctions.java b/atopcalcite/src/main/java/net/hydromatic/optiq/runtime/SqlFunctions.java new file mode 100644 index 0000000..f9ba797 --- /dev/null +++ b/atopcalcite/src/main/java/net/hydromatic/optiq/runtime/SqlFunctions.java @@ -0,0 +1,1705 @@ +/* + * OVERRIDE POINTS: + * - divide(BigDecimal,BigDecimal), was `b0.divide(b1)`, now `b0.divide(b1, MathContext.DECIMAL64);` + */ + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package net.hydromatic.optiq.runtime; + +import net.hydromatic.avatica.ByteString; + +import net.hydromatic.linq4j.Enumerable; +import net.hydromatic.linq4j.Linq4j; +import net.hydromatic.linq4j.expressions.Primitive; +import net.hydromatic.linq4j.function.Deterministic; +import net.hydromatic.linq4j.function.Function1; +import net.hydromatic.linq4j.function.NonDeterministic; + +import net.hydromatic.optiq.DataContext; + +import org.eigenbase.util14.DateTimeUtil; + +import java.math.*; +import java.sql.SQLException; +import java.sql.Timestamp; +import java.text.DecimalFormat; +import java.util.*; +import java.util.regex.Pattern; + +/** + * Helper methods to implement SQL functions in generated code. + * + *

Not present: and, or, not (builtin operators are better, because they + * use lazy evaluation. Implementations do not check for null values; the + * calling code must do that.

+ * + *

Many of the functions do not check for null values. This is intentional. + * If null arguments are possible, the code-generation framework checks for + * nulls before calling the functions.

+ */ +@SuppressWarnings({"unused", "rawtypes", "unchecked"}) +@Deterministic +public class SqlFunctions { + private static final DecimalFormat DOUBLE_FORMAT = + new DecimalFormat("0.0E0"); + + /** The julian date of the epoch, 1970-01-01. */ + public static final int EPOCH_JULIAN = 2440588; + + private static final TimeZone LOCAL_TZ = TimeZone.getDefault(); + + private static final Function1, Enumerable> + LIST_AS_ENUMERABLE = + new Function1, Enumerable>() { + public Enumerable apply(List list) { + return Linq4j.asEnumerable(list); + } + }; + + private SqlFunctions() { + } + + /** SQL SUBSTRING(string FROM ... FOR ...) function. */ + public static String substring(String s, int from, int for_) { + return s.substring(from - 1, Math.min(from - 1 + for_, s.length())); + } + + /** SQL SUBSTRING(string FROM ...) function. */ + public static String substring(String s, int from) { + return s.substring(from - 1); + } + + /** SQL UPPER(string) function. */ + public static String upper(String s) { + return s.toUpperCase(); + } + + /** SQL LOWER(string) function. */ + public static String lower(String s) { + return s.toLowerCase(); + } + + /** SQL INITCAP(string) function. */ + public static String initcap(String s) { + // Assumes Alpha as [A-Za-z0-9] + // white space is treated as everything else. + final int len = s.length(); + boolean start = true; + final StringBuilder newS = new StringBuilder(); + + for (int i = 0; i < len; i++) { + char curCh = s.charAt(i); + final int c = (int) curCh; + if (start) { // curCh is whitespace or first character of word. + if (c > 47 && c < 58) { // 0-9 + start = false; + } else if (c > 64 && c < 91) { // A-Z + start = false; + } else if (c > 96 && c < 123) { // a-z + start = false; + curCh = (char) (c - 32); // Uppercase this character + } + // else {} whitespace + } else { // Inside of a word or white space after end of word. + if (c > 47 && c < 58) { // 0-9 + // noop + } else if (c > 64 && c < 91) { // A-Z + curCh = (char) (c + 32); // Lowercase this character + } else if (c > 96 && c < 123) { // a-z + // noop + } else { // whitespace + start = true; + } + } + newS.append(curCh); + } // for each character in s + return newS.toString(); + } + + /** SQL CHARACTER_LENGTH(string) function. */ + public static int charLength(String s) { + return s.length(); + } + + /** SQL {@code string || string} operator. */ + public static String concat(String s0, String s1) { + return s0 + s1; + } + + /** SQL {@code binary || binary} operator. */ + public static ByteString concat(ByteString s0, ByteString s1) { + return s0.concat(s1); + } + + /** SQL {@code RTRIM} function applied to string. */ + public static String rtrim(String s) { + return trim_(s, false, true, ' '); + } + + /** SQL {@code LTRIM} function. */ + public static String ltrim(String s) { + return trim_(s, true, false, ' '); + } + + /** SQL {@code TRIM(... seek FROM s)} function. */ + public static String trim(boolean leading, boolean trailing, String seek, + String s) { + return trim_(s, leading, trailing, seek.charAt(0)); + } + + /** SQL {@code TRIM} function. */ + private static String trim_(String s, boolean left, boolean right, char c) { + int j = s.length(); + if (right) { + for (;;) { + if (j == 0) { + return ""; + } + if (s.charAt(j - 1) != c) { + break; + } + --j; + } + } + int i = 0; + if (left) { + for (;;) { + if (i == j) { + return ""; + } + if (s.charAt(i) != c) { + break; + } + ++i; + } + } + return s.substring(i, j); + } + + /** SQL {@code TRIM} function applied to binary string. */ + public static ByteString trim(ByteString s) { + return trim_(s, true, true); + } + + /** Helper for CAST. */ + public static ByteString rtrim(ByteString s) { + return trim_(s, false, true); + } + + /** SQL {@code TRIM} function applied to binary string. */ + private static ByteString trim_(ByteString s, boolean left, boolean right) { + int j = s.length(); + if (right) { + for (;;) { + if (j == 0) { + return ByteString.EMPTY; + } + if (s.byteAt(j - 1) != 0) { + break; + } + --j; + } + } + int i = 0; + if (left) { + for (;;) { + if (i == j) { + return ByteString.EMPTY; + } + if (s.byteAt(i) != 0) { + break; + } + ++i; + } + } + return s.substring(i, j); + } + + /** SQL {@code OVERLAY} function. */ + public static String overlay(String s, String r, int start) { + if (s == null || r == null) { + return null; + } + return s.substring(0, start - 1) + + r + + s.substring(start - 1 + r.length()); + } + + /** SQL {@code OVERLAY} function. */ + public static String overlay(String s, String r, int start, int length) { + if (s == null || r == null) { + return null; + } + return s.substring(0, start - 1) + + r + + s.substring(start - 1 + length); + } + + /** SQL {@code OVERLAY} function applied to binary strings. */ + public static ByteString overlay(ByteString s, ByteString r, int start) { + if (s == null || r == null) { + return null; + } + return s.substring(0, start - 1) + .concat(r) + .concat(s.substring(start - 1 + r.length())); + } + + /** SQL {@code OVERLAY} function applied to binary strings. */ + public static ByteString overlay(ByteString s, ByteString r, int start, + int length) { + if (s == null || r == null) { + return null; + } + return s.substring(0, start - 1) + .concat(r) + .concat(s.substring(start - 1 + length)); + } + + /** SQL {@code LIKE} function. */ + public static boolean like(String s, String pattern) { + final String regex = Like.sqlToRegexLike(pattern, null); + return Pattern.matches(regex, s); + } + + /** SQL {@code LIKE} function with escape. */ + public static boolean like(String s, String pattern, String escape) { + final String regex = Like.sqlToRegexLike(pattern, escape); + return Pattern.matches(regex, s); + } + + /** SQL {@code SIMILAR} function. */ + public static boolean similar(String s, String pattern) { + final String regex = Like.sqlToRegexSimilar(pattern, null); + return Pattern.matches(regex, s); + } + + /** SQL {@code SIMILAR} function with escape. */ + public static boolean similar(String s, String pattern, String escape) { + final String regex = Like.sqlToRegexSimilar(pattern, escape); + return Pattern.matches(regex, s); + } + + // = + + /** SQL = operator applied to Object values (including String; neither + * side may be null). */ + public static boolean eq(Object b0, Object b1) { + return b0.equals(b1); + } + + /** SQL = operator applied to BigDecimal values (neither may be null). */ + public static boolean eq(BigDecimal b0, BigDecimal b1) { + return b0.stripTrailingZeros().equals(b1.stripTrailingZeros()); + } + + // <> + + /** SQL <> operator applied to Object values (including String; + * neither side may be null). */ + public static boolean ne(Object b0, Object b1) { + return !b0.equals(b1); + } + + /** SQL <> operator applied to BigDecimal values. */ + public static boolean ne(BigDecimal b0, BigDecimal b1) { + return b0.compareTo(b1) != 0; + } + + // < + + /** SQL < operator applied to boolean values. */ + public static boolean lt(boolean b0, boolean b1) { + return compare(b0, b1) < 0; + } + + /** SQL < operator applied to String values. */ + public static boolean lt(String b0, String b1) { + return b0.compareTo(b1) < 0; + } + + /** SQL < operator applied to ByteString values. */ + public static boolean lt(ByteString b0, ByteString b1) { + return b0.compareTo(b1) < 0; + } + + /** SQL < operator applied to BigDecimal values. */ + public static boolean lt(BigDecimal b0, BigDecimal b1) { + return b0.compareTo(b1) < 0; + } + + // <= + + /** SQL ≤ operator applied to boolean values. */ + public static boolean le(boolean b0, boolean b1) { + return compare(b0, b1) <= 0; + } + + /** SQL ≤ operator applied to String values. */ + public static boolean le(String b0, String b1) { + return b0.compareTo(b1) <= 0; + } + + /** SQL ≤ operator applied to ByteString values. */ + public static boolean le(ByteString b0, ByteString b1) { + return b0.compareTo(b1) <= 0; + } + + /** SQL ≤ operator applied to BigDecimal values. */ + public static boolean le(BigDecimal b0, BigDecimal b1) { + return b0.compareTo(b1) <= 0; + } + + // > + + /** SQL > operator applied to boolean values. */ + public static boolean gt(boolean b0, boolean b1) { + return compare(b0, b1) > 0; + } + + /** SQL > operator applied to String values. */ + public static boolean gt(String b0, String b1) { + return b0.compareTo(b1) > 0; + } + + /** SQL > operator applied to ByteString values. */ + public static boolean gt(ByteString b0, ByteString b1) { + return b0.compareTo(b1) > 0; + } + + /** SQL > operator applied to BigDecimal values. */ + public static boolean gt(BigDecimal b0, BigDecimal b1) { + return b0.compareTo(b1) > 0; + } + + // >= + + /** SQL ≥ operator applied to boolean values. */ + public static boolean ge(boolean b0, boolean b1) { + return compare(b0, b1) >= 0; + } + + /** SQL ≥ operator applied to String values. */ + public static boolean ge(String b0, String b1) { + return b0.compareTo(b1) >= 0; + } + + /** SQL ≥ operator applied to ByteString values. */ + public static boolean ge(ByteString b0, ByteString b1) { + return b0.compareTo(b1) >= 0; + } + + /** SQL ≥ operator applied to BigDecimal values. */ + public static boolean ge(BigDecimal b0, BigDecimal b1) { + return b0.compareTo(b1) >= 0; + } + + // + + + /** SQL + operator applied to int values. */ + public static int plus(int b0, int b1) { + return b0 + b1; + } + + /** SQL + operator applied to int values; left side may be + * null. */ + public static Integer plus(Integer b0, int b1) { + return b0 == null ? null : (b0 + b1); + } + + /** SQL + operator applied to int values; right side may be + * null. */ + public static Integer plus(int b0, Integer b1) { + return b1 == null ? null : (b0 + b1); + } + + /** SQL + operator applied to nullable int values. */ + public static Integer plus(Integer b0, Integer b1) { + return (b0 == null || b1 == null) ? null : (b0 + b1); + } + + /** SQL + operator applied to nullable long and int values. */ + public static Long plus(Long b0, Integer b1) { + return (b0 == null || b1 == null) + ? null + : (b0.longValue() + b1.longValue()); + } + + /** SQL + operator applied to nullable int and long values. */ + public static Long plus(Integer b0, Long b1) { + return (b0 == null || b1 == null) + ? null + : (b0.longValue() + b1.longValue()); + } + + /** SQL + operator applied to BigDecimal values. */ + public static BigDecimal plus(BigDecimal b0, BigDecimal b1) { + return (b0 == null || b1 == null) ? null : b0.add(b1); + } + + // - + + /** SQL - operator applied to int values. */ + public static int minus(int b0, int b1) { + return b0 - b1; + } + + /** SQL - operator applied to int values; left side may be + * null. */ + public static Integer minus(Integer b0, int b1) { + return b0 == null ? null : (b0 - b1); + } + + /** SQL - operator applied to int values; right side may be + * null. */ + public static Integer minus(int b0, Integer b1) { + return b1 == null ? null : (b0 - b1); + } + + /** SQL - operator applied to nullable int values. */ + public static Integer minus(Integer b0, Integer b1) { + return (b0 == null || b1 == null) ? null : (b0 - b1); + } + + /** SQL - operator applied to nullable long and int values. */ + public static Long minus(Long b0, Integer b1) { + return (b0 == null || b1 == null) + ? null + : (b0.longValue() - b1.longValue()); + } + + /** SQL - operator applied to nullable int and long values. */ + public static Long minus(Integer b0, Long b1) { + return (b0 == null || b1 == null) + ? null + : (b0.longValue() - b1.longValue()); + } + + /** SQL - operator applied to BigDecimal values. */ + public static BigDecimal minus(BigDecimal b0, BigDecimal b1) { + return (b0 == null || b1 == null) ? null : b0.subtract(b1); + } + + // / + + /** SQL / operator applied to int values. */ + public static int divide(int b0, int b1) { + return b0 / b1; + } + + /** SQL / operator applied to int values; left side may be + * null. */ + public static Integer divide(Integer b0, int b1) { + return b0 == null ? null : (b0 / b1); + } + + /** SQL / operator applied to int values; right side may be + * null. */ + public static Integer divide(int b0, Integer b1) { + return b1 == null ? null : (b0 / b1); + } + + /** SQL / operator applied to nullable int values. */ + public static Integer divide(Integer b0, Integer b1) { + return (b0 == null || b1 == null) ? null : (b0 / b1); + } + + /** SQL / operator applied to nullable long and int values. */ + public static Long divide(Long b0, Integer b1) { + return (b0 == null || b1 == null) + ? null + : (b0.longValue() / b1.longValue()); + } + + /** SQL / operator applied to nullable int and long values. */ + public static Long divide(Integer b0, Long b1) { + return (b0 == null || b1 == null) + ? null + : (b0.longValue() / b1.longValue()); + } + + /** SQL / operator applied to BigDecimal values. */ + public static BigDecimal divide(BigDecimal b0, BigDecimal b1) { + // OVERRIDE POINT + return (b0 == null || b1 == null) ? null : b0.divide(b1, MathContext.DECIMAL64); + } + + // * + + /** SQL * operator applied to int values. */ + public static int multiply(int b0, int b1) { + return b0 * b1; + } + + /** SQL * operator applied to int values; left side may be + * null. */ + public static Integer multiply(Integer b0, int b1) { + return b0 == null ? null : (b0 * b1); + } + + /** SQL * operator applied to int values; right side may be + * null. */ + public static Integer multiply(int b0, Integer b1) { + return b1 == null ? null : (b0 * b1); + } + + /** SQL * operator applied to nullable int values. */ + public static Integer multiply(Integer b0, Integer b1) { + return (b0 == null || b1 == null) ? null : (b0 * b1); + } + + /** SQL * operator applied to nullable long and int values. */ + public static Long multiply(Long b0, Integer b1) { + return (b0 == null || b1 == null) + ? null + : (b0.longValue() * b1.longValue()); + } + + /** SQL * operator applied to nullable int and long values. */ + public static Long multiply(Integer b0, Long b1) { + return (b0 == null || b1 == null) + ? null + : (b0.longValue() * b1.longValue()); + } + + /** SQL * operator applied to BigDecimal values. */ + public static BigDecimal multiply(BigDecimal b0, BigDecimal b1) { + return (b0 == null || b1 == null) ? null : b0.multiply(b1); + } + + // EXP + + /** SQL EXP operator applied to double values. */ + public static double exp(double b0) { + return Math.exp(b0); + } + + public static double exp(long b0) { + return Math.exp(b0); + } + + // POWER + + /** SQL POWER operator applied to double values. */ + public static double power(double b0, double b1) { + return Math.pow(b0, b1); + } + + public static double power(long b0, long b1) { + return Math.pow(b0, b1); + } + + public static double power(long b0, BigDecimal b1) { + return Math.pow(b0, b1.doubleValue()); + } + + // LN + + /** SQL {@code LN(number)} function applied to double values. */ + public static double ln(double d) { + return Math.log(d); + } + + /** SQL {@code LN(number)} function applied to long values. */ + public static double ln(long b0) { + return Math.log(b0); + } + + /** SQL {@code LN(number)} function applied to BigDecimal values. */ + public static double ln(BigDecimal d) { + return Math.log(d.doubleValue()); + } + + // LOG10 + + /** SQL LOG10(numeric) operator applied to double values. */ + public static double log10(double b0) { + return Math.log10(b0); + } + + /** SQL {@code LOG10(number)} function applied to long values. */ + public static double log10(long b0) { + return Math.log10(b0); + } + + /** SQL {@code LOG10(number)} function applied to BigDecimal values. */ + public static double log10(BigDecimal d) { + return Math.log10(d.doubleValue()); + } + + // MOD + + /** SQL MOD operator applied to byte values. */ + public static byte mod(byte b0, byte b1) { + return (byte) (b0 % b1); + } + + /** SQL MOD operator applied to short values. */ + public static short mod(short b0, short b1) { + return (short) (b0 % b1); + } + + /** SQL MOD operator applied to int values. */ + public static int mod(int b0, int b1) { + return b0 % b1; + } + + /** SQL MOD operator applied to long values. */ + public static long mod(long b0, long b1) { + return b0 % b1; + } + + // temporary + public static BigDecimal mod(BigDecimal b0, int b1) { + return mod(b0, BigDecimal.valueOf(b1)); + } + + // temporary + public static int mod(int b0, BigDecimal b1) { + return mod(b0, b1.intValue()); + } + + public static BigDecimal mod(BigDecimal b0, BigDecimal b1) { + final BigDecimal[] bigDecimals = b0.divideAndRemainder(b1); + return bigDecimals[1]; + } + + // ABS + + /** SQL ABS operator applied to byte values. */ + public static byte abs(byte b0) { + return (byte) Math.abs(b0); + } + + /** SQL ABS operator applied to short values. */ + public static short abs(short b0) { + return (short) Math.abs(b0); + } + + /** SQL ABS operator applied to int values. */ + public static int abs(int b0) { + return Math.abs(b0); + } + + /** SQL ABS operator applied to long values. */ + public static long abs(long b0) { + return Math.abs(b0); + } + + /** SQL ABS operator applied to float values. */ + public static float abs(float b0) { + return Math.abs(b0); + } + + /** SQL ABS operator applied to double values. */ + public static double abs(double b0) { + return Math.abs(b0); + } + + /** SQL ABS operator applied to BigDecimal values. */ + public static BigDecimal abs(BigDecimal b0) { + return b0.abs(); + } + + // Helpers + + /** Helper for implementing MIN. Somewhat similar to LEAST operator. */ + public static > T lesser(T b0, T b1) { + return b0 == null || b0.compareTo(b1) > 0 ? b1 : b0; + } + + /** LEAST operator. */ + public static > T least(T b0, T b1) { + return b0 == null || b1 != null && b0.compareTo(b1) > 0 ? b1 : b0; + } + + public static boolean greater(boolean b0, boolean b1) { + return b0 || b1; + } + + public static boolean lesser(boolean b0, boolean b1) { + return b0 && b1; + } + + public static byte greater(byte b0, byte b1) { + return b0 > b1 ? b0 : b1; + } + + public static byte lesser(byte b0, byte b1) { + return b0 > b1 ? b1 : b0; + } + + public static char greater(char b0, char b1) { + return b0 > b1 ? b0 : b1; + } + + public static char lesser(char b0, char b1) { + return b0 > b1 ? b1 : b0; + } + + public static short greater(short b0, short b1) { + return b0 > b1 ? b0 : b1; + } + + public static short lesser(short b0, short b1) { + return b0 > b1 ? b1 : b0; + } + + public static int greater(int b0, int b1) { + return b0 > b1 ? b0 : b1; + } + + public static int lesser(int b0, int b1) { + return b0 > b1 ? b1 : b0; + } + + public static long greater(long b0, long b1) { + return b0 > b1 ? b0 : b1; + } + + public static long lesser(long b0, long b1) { + return b0 > b1 ? b1 : b0; + } + + public static float greater(float b0, float b1) { + return b0 > b1 ? b0 : b1; + } + + public static float lesser(float b0, float b1) { + return b0 > b1 ? b1 : b0; + } + + public static double greater(double b0, double b1) { + return b0 > b1 ? b0 : b1; + } + + public static double lesser(double b0, double b1) { + return b0 > b1 ? b1 : b0; + } + + /** Helper for implementing MAX. Somewhat similar to GREATEST operator. */ + public static > T greater(T b0, T b1) { + return b0 == null || b0.compareTo(b1) < 0 ? b1 : b0; + } + + /** GREATEST operator. */ + public static > T greatest(T b0, T b1) { + return b0 == null || b1 != null && b0.compareTo(b1) < 0 ? b1 : b0; + } + + /** Boolean comparison. */ + public static int compare(boolean x, boolean y) { + return x == y ? 0 : x ? 1 : -1; + } + + /** CAST(FLOAT AS VARCHAR). */ + public static String toString(float x) { + if (x == 0) { + return "0E0"; + } + BigDecimal bigDecimal = + new BigDecimal(x, MathContext.DECIMAL32).stripTrailingZeros(); + final String s = bigDecimal.toString(); + return s.replaceAll("0*E", "E").replace("E+", "E"); + } + + /** CAST(DOUBLE AS VARCHAR). */ + public static String toString(double x) { + if (x == 0) { + return "0E0"; + } + BigDecimal bigDecimal = + new BigDecimal(x, MathContext.DECIMAL64).stripTrailingZeros(); + final String s = bigDecimal.toString(); + return s.replaceAll("0*E", "E").replace("E+", "E"); + } + + /** CAST(DECIMAL AS VARCHAR). */ + public static String toString(BigDecimal x) { + final String s = x.toString(); + if (s.startsWith("0")) { + // we want ".1" not "0.1" + return s.substring(1); + } else if (s.startsWith("-0")) { + // we want "-.1" not "-0.1" + return "-" + s.substring(2); + } else { + return s; + } + } + + /** CAST(BOOLEAN AS VARCHAR). */ + public static String toString(boolean x) { + // Boolean.toString returns lower case -- no good. + return x ? "TRUE" : "FALSE"; + } + + @NonDeterministic + private static Object cannotConvert(Object o, Class toType) { + throw new RuntimeException("Cannot convert " + o + " to " + toType); + } + + /** CAST(VARCHAR AS BOOLEAN). */ + public static boolean toBoolean(String s) { + s = trim_(s, true, true, ' '); + if (s.equalsIgnoreCase("TRUE")) { + return true; + } else if (s.equalsIgnoreCase("FALSE")) { + return false; + } else { + throw new RuntimeException("Invalid character for cast"); + } + } + + public static boolean toBoolean(Number number) { + return !number.equals(0); + } + + public static boolean toBoolean(Object o) { + return o instanceof Boolean ? (Boolean) o + : o instanceof Number ? toBoolean((Number) o) + : o instanceof String ? toBoolean((String) o) + : (Boolean) cannotConvert(o, boolean.class); + } + + // Don't need parseByte etc. - Byte.parseByte is sufficient. + + public static byte toByte(Object o) { + return o instanceof Byte ? (Byte) o + : o instanceof Number ? toByte((Number) o) + : Byte.parseByte(o.toString()); + } + + public static byte toByte(Number number) { + return number.byteValue(); + } + + public static char toChar(String s) { + return s.charAt(0); + } + + public static Character toCharBoxed(String s) { + return s.charAt(0); + } + + public static short toShort(String s) { + return Short.parseShort(s.trim()); + } + + public static short toShort(Number number) { + return number.shortValue(); + } + + public static short toShort(Object o) { + return o instanceof Short ? (Short) o + : o instanceof Number ? toShort((Number) o) + : o instanceof String ? toShort((String) o) + : (Short) cannotConvert(o, short.class); + } + + public static int toInt(java.util.Date v) { + return toInt(v, LOCAL_TZ); + } + + public static int toInt(java.util.Date v, TimeZone timeZone) { + return (int) (toLong(v, timeZone) / DateTimeUtil.MILLIS_PER_DAY); + } + + public static Integer toIntOptional(java.util.Date v) { + return v == null ? null : toInt(v); + } + + public static Integer toIntOptional(java.util.Date v, TimeZone timeZone) { + return v == null + ? null + : toInt(v, timeZone); + } + + public static long toLong(Date v) { + return toLong(v, LOCAL_TZ); + } + + public static int toInt(java.sql.Time v) { + return (int) (toLong(v) % DateTimeUtil.MILLIS_PER_DAY); + } + + public static Integer toIntOptional(java.sql.Time v) { + return v == null ? null : toInt(v); + } + + public static int toInt(String s) { + return Integer.parseInt(s.trim()); + } + + public static int toInt(Number number) { + return number.intValue(); + } + + public static int toInt(Object o) { + return o instanceof Integer ? (Integer) o + : o instanceof Number ? toInt((Number) o) + : o instanceof String ? toInt((String) o) + : (Integer) cannotConvert(o, int.class); + } + + public static long toLong(Timestamp v) { + return toLong(v, LOCAL_TZ); + } + + // mainly intended for java.sql.Timestamp but works for other dates also + public static long toLong(java.util.Date v, TimeZone timeZone) { + final long time = v.getTime(); + return time + timeZone.getOffset(time); + } + + // mainly intended for java.sql.Timestamp but works for other dates also + public static Long toLongOptional(java.util.Date v) { + return v == null ? null : toLong(v, LOCAL_TZ); + } + + public static Long toLongOptional(Timestamp v, TimeZone timeZone) { + if (v == null) { + return null; + } + return toLong(v, LOCAL_TZ); + } + + public static long toLong(String s) { + if (s.startsWith("199") && s.contains(":")) { + return Timestamp.valueOf(s).getTime(); + } + return Long.parseLong(s.trim()); + } + + public static long toLong(Number number) { + return number.longValue(); + } + + public static long toLong(Object o) { + return o instanceof Long ? (Long) o + : o instanceof Number ? toLong((Number) o) + : o instanceof String ? toLong((String) o) + : (Long) cannotConvert(o, long.class); + } + + public static float toFloat(String s) { + return Float.parseFloat(s.trim()); + } + + public static float toFloat(Number number) { + return number.floatValue(); + } + + public static float toFloat(Object o) { + return o instanceof Float ? (Float) o + : o instanceof Number ? toFloat((Number) o) + : o instanceof String ? toFloat((String) o) + : (Float) cannotConvert(o, float.class); + } + + public static double toDouble(String s) { + return Double.parseDouble(s.trim()); + } + + public static double toDouble(Number number) { + return number.doubleValue(); + } + + public static double toDouble(Object o) { + return o instanceof Double ? (Double) o + : o instanceof Number ? toDouble((Number) o) + : o instanceof String ? toDouble((String) o) + : (Double) cannotConvert(o, double.class); + } + + public static BigDecimal toBigDecimal(String s) { + return new BigDecimal(s.trim()); + } + + public static BigDecimal toBigDecimal(Number number) { + // There are some values of "long" that cannot be represented as "double". + // Not so "int". If it isn't a long, go straight to double. + return number instanceof BigDecimal ? (BigDecimal) number + : number instanceof BigInteger ? new BigDecimal((BigInteger) number) + : number instanceof Long ? new BigDecimal(number.longValue()) + : new BigDecimal(number.doubleValue()); + } + + public static BigDecimal toBigDecimal(Object o) { + return o instanceof Number ? toBigDecimal((Number) o) + : toBigDecimal(o.toString()); + } + + // Don't need shortValueOf etc. - Short.valueOf is sufficient. + + /** Helper for CAST(... AS VARCHAR(maxLength)). */ + public static String truncate(String s, int maxLength) { + return s == null ? null + : s.length() > maxLength ? s.substring(0, maxLength) + : s; + } + + /** Helper for CAST(... AS VARBINARY(maxLength)). */ + public static ByteString truncate(ByteString s, int maxLength) { + return s == null ? null + : s.length() > maxLength ? s.substring(0, maxLength) + : s; + } + + /** SQL {@code POSITION(seek IN string)} function. */ + public static int position(String seek, String s) { + return s.indexOf(seek) + 1; + } + + /** SQL {@code POSITION(seek IN string)} function. */ + public static int position(ByteString seek, ByteString s) { + return s.indexOf(seek) + 1; + } + + /** Cheap, unsafe, long power. power(2, 3) returns 8. */ + public static long powerX(long a, long b) { + long x = 1; + while (b > 0) { + x *= a; + --b; + } + return x; + } + + /** Helper for rounding. Truncate(12345, 1000) returns 12000. */ + public static long round(long v, long x) { + return truncate(v + x / 2, x); + } + + /** Helper for rounding. Truncate(12345, 1000) returns 12000. */ + public static long truncate(long v, long x) { + long remainder = v % x; + if (remainder < 0) { + remainder += x; + } + return v - remainder; + } + + /** Helper for rounding. Truncate(12345, 1000) returns 12000. */ + public static int round(int v, int x) { + return truncate(v + x / 2, x); + } + + /** Helper for rounding. Truncate(12345, 1000) returns 12000. */ + public static int truncate(int v, int x) { + int remainder = v % x; + if (remainder < 0) { + remainder += x; + } + return v - remainder; + } + + /** Helper for CAST({timestamp} AS VARCHAR(n)). */ + public static String unixTimestampToString(long timestamp) { + final StringBuilder buf = new StringBuilder(17); + int date = (int) (timestamp / DateTimeUtil.MILLIS_PER_DAY); + int time = (int) (timestamp % DateTimeUtil.MILLIS_PER_DAY); + if (time < 0) { + --date; + time += DateTimeUtil.MILLIS_PER_DAY; + } + unixDateToString(buf, date); + buf.append(' '); + unixTimeToString(buf, time); + return buf.toString(); + } + + /** Helper for CAST({timestamp} AS VARCHAR(n)). */ + public static String unixTimeToString(int time) { + final StringBuilder buf = new StringBuilder(8); + unixTimeToString(buf, time); + return buf.toString(); + } + + private static void unixTimeToString(StringBuilder buf, int time) { + int h = time / 3600000; + int time2 = time % 3600000; + int m = time2 / 60000; + int time3 = time2 % 60000; + int s = time3 / 1000; + int ms = time3 % 1000; + int2(buf, h); + buf.append(':'); + int2(buf, m); + buf.append(':'); + int2(buf, s); + } + + /** SQL {@code CURRENT_TIMESTAMP} function. */ + @NonDeterministic + public static long currentTimestamp(DataContext root) { + // Cast required for JDK 1.6. + return (Long) DataContext.Variable.CURRENT_TIMESTAMP.get(root); + } + + /** SQL {@code CURRENT_TIME} function. */ + @NonDeterministic + public static int currentTime(DataContext root) { + int time = (int) (currentTimestamp(root) % DateTimeUtil.MILLIS_PER_DAY); + if (time < 0) { + time += DateTimeUtil.MILLIS_PER_DAY; + } + return time; + } + + /** SQL {@code CURRENT_DATE} function. */ + @NonDeterministic + public static int currentDate(DataContext root) { + final long timestamp = currentTimestamp(root); + int date = (int) (timestamp / DateTimeUtil.MILLIS_PER_DAY); + final int time = (int) (timestamp % DateTimeUtil.MILLIS_PER_DAY); + if (time < 0) { + --date; + } + return date; + } + + /** SQL {@code LOCAL_TIMESTAMP} function. */ + @NonDeterministic + public static long localTimestamp(DataContext root) { + // Cast required for JDK 1.6. + return (Long) DataContext.Variable.LOCAL_TIMESTAMP.get(root); + } + + /** SQL {@code LOCAL_TIME} function. */ + @NonDeterministic + public static int localTime(DataContext root) { + return (int) (localTimestamp(root) % DateTimeUtil.MILLIS_PER_DAY); + } + + private static void int2(StringBuilder buf, int i) { + buf.append((char) ('0' + (i / 10) % 10)); + buf.append((char) ('0' + i % 10)); + } + + private static void int4(StringBuilder buf, int i) { + buf.append((char) ('0' + (i / 1000) % 10)); + buf.append((char) ('0' + (i / 100) % 10)); + buf.append((char) ('0' + (i / 10) % 10)); + buf.append((char) ('0' + i % 10)); + } + + public static int dateStringToUnixDate(String s) { + int hyphen1 = s.indexOf('-'); + int y; + int m; + int d; + if (hyphen1 < 0) { + y = Integer.parseInt(s.trim()); + m = 1; + d = 1; + } else { + y = Integer.parseInt(s.substring(0, hyphen1).trim()); + final int hyphen2 = s.indexOf('-', hyphen1 + 1); + if (hyphen2 < 0) { + m = Integer.parseInt(s.substring(hyphen1 + 1).trim()); + d = 1; + } else { + m = Integer.parseInt(s.substring(hyphen1 + 1, hyphen2).trim()); + d = Integer.parseInt(s.substring(hyphen2 + 1).trim()); + } + } + return ymdToUnixDate(y, m, d); + } + + public static int timeStringToUnixDate(String v) { + return timeStringToUnixDate(v, 0); + } + + public static int timeStringToUnixDate(String v, int start) { + final int colon1 = v.indexOf(':', start); + int hour; + int minute; + int second; + int milli; + if (colon1 < 0) { + hour = Integer.parseInt(v.trim()); + minute = 1; + second = 1; + milli = 0; + } else { + hour = Integer.parseInt(v.substring(start, colon1).trim()); + final int colon2 = v.indexOf(':', colon1 + 1); + if (colon2 < 0) { + minute = Integer.parseInt(v.substring(colon1 + 1).trim()); + second = 1; + milli = 0; + } else { + minute = Integer.parseInt(v.substring(colon1 + 1, colon2).trim()); + int dot = v.indexOf('.', colon2); + if (dot < 0) { + second = Integer.parseInt(v.substring(colon2 + 1).trim()); + milli = 0; + } else { + second = Integer.parseInt(v.substring(colon2 + 1, dot).trim()); + milli = Integer.parseInt(v.substring(dot + 1).trim()); + } + } + } + return hour * (int) DateTimeUtil.MILLIS_PER_HOUR + + minute * (int) DateTimeUtil.MILLIS_PER_MINUTE + + second * (int) DateTimeUtil.MILLIS_PER_SECOND + + milli; + } + + public static long timestampStringToUnixDate(String s) { + final long d; + final long t; + s = s.trim(); + int space = s.indexOf(' '); + if (space >= 0) { + d = dateStringToUnixDate(s.substring(0, space)); + t = timeStringToUnixDate(s, space + 1); + } else { + d = dateStringToUnixDate(s); + t = 0; + } + return d * DateTimeUtil.MILLIS_PER_DAY + t; + } + + /** Helper for CAST({date} AS VARCHAR(n)). */ + public static String unixDateToString(int date) { + final StringBuilder buf = new StringBuilder(10); + unixDateToString(buf, date); + return buf.toString(); + } + + private static void unixDateToString(StringBuilder buf, int date) { + julianToString(buf, date + EPOCH_JULIAN); + } + + private static void julianToString(StringBuilder buf, int julian) { + // this shifts the epoch back to astronomical year -4800 instead of the + // start of the Christian era in year AD 1 of the proleptic Gregorian + // calendar. + int j = julian + 32044; + int g = j / 146097; + int dg = j % 146097; + int c = (dg / 36524 + 1) * 3 / 4; + int dc = dg - c * 36524; + int b = dc / 1461; + int db = dc % 1461; + int a = (db / 365 + 1) * 3 / 4; + int da = db - a * 365; + + // integer number of full years elapsed since March 1, 4801 BC + int y = g * 400 + c * 100 + b * 4 + a; + // integer number of full months elapsed since the last March 1 + int m = (da * 5 + 308) / 153 - 2; + // number of days elapsed since day 1 of the month + int d = da - (m + 4) * 153 / 5 + 122; + int year = y - 4800 + (m + 2) / 12; + int month = (m + 2) % 12 + 1; + int day = d + 1; + int4(buf, year); + buf.append('-'); + int2(buf, month); + buf.append('-'); + int2(buf, day); + } + + public static long unixDateExtract(TimeUnitRange range, long date) { + return julianExtract(range, (int) date + EPOCH_JULIAN); + } + + private static int julianExtract(TimeUnitRange range, int julian) { + // this shifts the epoch back to astronomical year -4800 instead of the + // start of the Christian era in year AD 1 of the proleptic Gregorian + // calendar. + int j = julian + 32044; + int g = j / 146097; + int dg = j % 146097; + int c = (dg / 36524 + 1) * 3 / 4; + int dc = dg - c * 36524; + int b = dc / 1461; + int db = dc % 1461; + int a = (db / 365 + 1) * 3 / 4; + int da = db - a * 365; + + // integer number of full years elapsed since March 1, 4801 BC + int y = g * 400 + c * 100 + b * 4 + a; + // integer number of full months elapsed since the last March 1 + int m = (da * 5 + 308) / 153 - 2; + // number of days elapsed since day 1 of the month + int d = da - (m + 4) * 153 / 5 + 122; + int year = y - 4800 + (m + 2) / 12; + int month = (m + 2) % 12 + 1; + int day = d + 1; + switch (range) { + case YEAR: + return year; + case MONTH: + return month; + case DAY: + return day; + default: + throw new AssertionError(range); + } + } + + public static int ymdToUnixDate(int year, int month, int day) { + final int julian = ymdToJulian(year, month, day); + return julian - EPOCH_JULIAN; + } + + public static int ymdToJulian(int year, int month, int day) { + int a = (14 - month) / 12; + int y = year + 4800 - a; + int m = month + 12 * a - 3; + int j = day + (153 * m + 2) / 5 + + 365 * y + + y / 4 + - y / 100 + + y / 400 + - 32045; + if (j < 2299161) { + j = day + (153 * m + 2) / 5 + 365 * y + y / 4 - 32083; + } + return j; + } + + public static String intervalYearMonthToString(int v, TimeUnitRange range) { + final StringBuilder buf = new StringBuilder(); + if (v >= 0) { + buf.append('+'); + } else { + buf.append('-'); + v = -v; + } + final int y; + final int m; + switch (range) { + case YEAR: + v = roundUp(v, 12); + y = v / 12; + buf.append(y); + break; + case YEAR_TO_MONTH: + y = v / 12; + buf.append(y); + buf.append('-'); + m = v % 12; + number(buf, m, 2); + break; + case MONTH: + m = v; + buf.append(m); + break; + default: + throw new AssertionError(range); + } + return buf.toString(); + } + + private static StringBuilder number(StringBuilder buf, int v, int n) { + for (int k = digitCount(v); k < n; k++) { + buf.append('0'); + } + return buf.append(v); + } + + public static int digitCount(int v) { + for (int n = 1;; n++) { + v /= 10; + if (v == 0) { + return n; + } + } + } + + public static String intervalDayTimeToString(long v, TimeUnitRange range, + int scale) { + final StringBuilder buf = new StringBuilder(); + if (v >= 0) { + buf.append('+'); + } else { + buf.append('-'); + v = -v; + } + final long ms; + final long s; + final long m; + final long h; + final long d; + switch (range) { + case DAY_TO_SECOND: + v = roundUp(v, powerX(10, 3 - scale)); + ms = v % 1000; + v /= 1000; + s = v % 60; + v /= 60; + m = v % 60; + v /= 60; + h = v % 24; + v /= 24; + d = v; + buf.append((int) d); + buf.append(' '); + number(buf, (int) h, 2); + buf.append(':'); + number(buf, (int) m, 2); + buf.append(':'); + number(buf, (int) s, 2); + fraction(buf, scale, ms); + break; + case DAY_TO_MINUTE: + v = roundUp(v, 1000 * 60); + v /= 1000; + v /= 60; + m = v % 60; + v /= 60; + h = v % 24; + v /= 24; + d = v; + buf.append((int) d); + buf.append(' '); + number(buf, (int) h, 2); + buf.append(':'); + number(buf, (int) m, 2); + break; + case DAY_TO_HOUR: + v = roundUp(v, 1000 * 60 * 60); + v /= 1000; + v /= 60; + v /= 60; + h = v % 24; + v /= 24; + d = v; + buf.append((int) d); + buf.append(' '); + number(buf, (int) h, 2); + break; + case DAY: + v = roundUp(v, 1000 * 60 * 60 * 24); + d = v / (1000 * 60 * 60 * 24); + buf.append((int) d); + break; + case HOUR: + v = roundUp(v, 1000 * 60 * 60); + v /= 1000; + v /= 60; + v /= 60; + h = v; + buf.append((int) h); + break; + case HOUR_TO_MINUTE: + v = roundUp(v, 1000 * 60); + v /= 1000; + v /= 60; + m = v % 60; + v /= 60; + h = v; + buf.append((int) h); + buf.append(':'); + number(buf, (int) m, 2); + break; + case HOUR_TO_SECOND: + v = roundUp(v, powerX(10, 3 - scale)); + ms = v % 1000; + v /= 1000; + s = v % 60; + v /= 60; + m = v % 60; + v /= 60; + h = v; + buf.append((int) h); + buf.append(':'); + number(buf, (int) m, 2); + buf.append(':'); + number(buf, (int) s, 2); + fraction(buf, scale, ms); + break; + case MINUTE_TO_SECOND: + v = roundUp(v, powerX(10, 3 - scale)); + ms = v % 1000; + v /= 1000; + s = v % 60; + v /= 60; + m = v; + buf.append((int) m); + buf.append(':'); + number(buf, (int) s, 2); + fraction(buf, scale, ms); + break; + case MINUTE: + v = roundUp(v, 1000 * 60); + v /= 1000; + v /= 60; + m = v; + buf.append((int) m); + break; + case SECOND: + v = roundUp(v, powerX(10, 3 - scale)); + ms = v % 1000; + v /= 1000; + s = v; + buf.append((int) s); + fraction(buf, scale, ms); + break; + default: + throw new AssertionError(range); + } + return buf.toString(); + } + + /** + * Rounds a dividend to the nearest divisor. + * For example roundUp(31, 10) yields 30; roundUp(37, 10) yields 40. + * @param dividend Number to be divided + * @param divisor Number to divide by + * @return Rounded dividend + */ + private static long roundUp(long dividend, long divisor) { + long remainder = dividend % divisor; + dividend -= remainder; + if (remainder * 2 > divisor) { + dividend += divisor; + } + return dividend; + } + + private static int roundUp(int dividend, int divisor) { + int remainder = dividend % divisor; + dividend -= remainder; + if (remainder * 2 > divisor) { + dividend += divisor; + } + return dividend; + } + + private static void fraction(StringBuilder buf, int scale, long ms) { + if (scale > 0) { + buf.append('.'); + long v1 = scale == 3 ? ms + : scale == 2 ? ms / 10 + : scale == 1 ? ms / 100 + : 0; + number(buf, (int) v1, scale); + } + } + + /** Helper for "array element reference". Caller has already ensured that + * array and index are not null. Index is 1-based, per SQL. */ + public static Object arrayItem(List list, int item) { + if (item < 1 || item > list.size()) { + return null; + } + return list.get(item - 1); + } + + /** Helper for "map element reference". Caller has already ensured that + * array and index are not null. Index is 1-based, per SQL. */ + public static Object mapItem(Map map, Object item) { + return map.get(item); + } + + /** Implements the {@code [ ... ]} operator on an object whose type is not + * known until runtime. + */ + public static Object item(Object object, Object index) { + if (object instanceof Map) { + return ((Map) object).get(index); + } + if (object instanceof List && index instanceof Number) { + List list = (List) object; + return list.get(((Number) index).intValue()); + } + return null; + } + + /** NULL → FALSE, FALSE → FALSE, TRUE → TRUE. */ + public static boolean isTrue(Boolean b) { + return b != null && b; + } + + /** NULL → TRUE, FALSE → FALSE, TRUE → TRUE. */ + public static boolean isNotFalse(Boolean b) { + return b == null || b; + } + + /** NULL → NULL, FALSE → TRUE, TRUE → FALSE. */ + public static Boolean not(Boolean b) { + return (b == null) ? null : !b; + } + + /** Converts a JDBC array to a list. */ + public static List arrayToList(final java.sql.Array a) { + if (a == null) { + return null; + } + try { + return Primitive.asList(a.getArray()); + } catch (SQLException e) { + throw new RuntimeException(e); + } + } + + /** Support the SLICE function. */ + public static List slice(List list) { + return list; + } + + /** Support the ELEMENT function. */ + public static Object element(List list) { + switch (list.size()) { + case 0: + return null; + case 1: + return list.get(0); + default: + throw new RuntimeException("more than one value"); + } + } + + /** Returns a lambda that converts a list to an enumerable. */ + public static Function1, Enumerable> listToEnumerable() { + //noinspection unchecked + return (Function1, Enumerable>) (Function1) LIST_AS_ENUMERABLE; + } + + /** A range of time units. The first is more significant than the + * other (e.g. year-to-day) or the same as the other + * (e.g. month). */ + public enum TimeUnitRange { + YEAR, + YEAR_TO_MONTH, + MONTH, + DAY, + DAY_TO_HOUR, + DAY_TO_MINUTE, + DAY_TO_SECOND, + HOUR, + HOUR_TO_MINUTE, + HOUR_TO_SECOND, + MINUTE, + MINUTE_TO_SECOND, + SECOND; + + /** Whether this is in the YEAR-TO-MONTH family of intervals. */ + public boolean monthly() { + return ordinal() <= MONTH.ordinal(); + } + } +} + +// End SqlFunctions.java diff --git a/atopcalcite/src/main/java/org/eigenbase/sql2rel/SqlToRelConverter.java b/atopcalcite/src/main/java/org/eigenbase/sql2rel/SqlToRelConverter.java new file mode 100644 index 0000000..15077e0 --- /dev/null +++ b/atopcalcite/src/main/java/org/eigenbase/sql2rel/SqlToRelConverter.java @@ -0,0 +1,4782 @@ +/* + * OVERRIDE POINTS: + * - getInSubqueryThreshold(), was `20`, now `Integer.MAX_VALUE` + * - isTrimUnusedFields(), override to false + */ + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.eigenbase.sql2rel; + +import java.lang.reflect.Type; +import java.math.*; +import java.util.*; +import java.util.logging.*; + +import org.eigenbase.rel.*; +import org.eigenbase.rel.metadata.*; +import org.eigenbase.relopt.*; +import org.eigenbase.reltype.*; +import org.eigenbase.rex.*; +import org.eigenbase.sql.*; +import org.eigenbase.sql.fun.*; +import org.eigenbase.sql.parser.*; +import org.eigenbase.sql.type.*; +import org.eigenbase.sql.util.*; +import org.eigenbase.sql.validate.*; +import org.eigenbase.trace.*; +import org.eigenbase.util.*; +import org.eigenbase.util.mapping.Mappings; +import org.eigenbase.util14.*; + +import net.hydromatic.linq4j.Ord; +import net.hydromatic.optiq.ModifiableTable; +import net.hydromatic.optiq.TranslatableTable; +import net.hydromatic.optiq.prepare.Prepare; +import net.hydromatic.optiq.prepare.RelOptTableImpl; +import net.hydromatic.optiq.util.BitSets; + +import com.google.common.base.Function; +import com.google.common.collect.*; + +import static org.eigenbase.sql.SqlUtil.stripAs; +import static org.eigenbase.util.Static.RESOURCE; + +/** + * Converts a SQL parse tree (consisting of {@link org.eigenbase.sql.SqlNode} + * objects) into a relational algebra expression (consisting of + * {@link org.eigenbase.rel.RelNode} objects). + * + *

The public entry points are: {@link #convertQuery}, + * {@link #convertExpression(SqlNode)}. + */ +@SuppressWarnings({"unused", "rawtypes", "unchecked", "incomplete-switch", "deprecation"}) +public class SqlToRelConverter { + //~ Static fields/initializers --------------------------------------------- + + protected static final Logger SQL2REL_LOGGER = + EigenbaseTrace.getSqlToRelTracer(); + + private static final Function FN = + new Function() { + public SqlNode apply(SubQuery input) { + return input.node; + } + }; + + //~ Instance fields -------------------------------------------------------- + + protected final SqlValidator validator; + protected final RexBuilder rexBuilder; + protected final Prepare.CatalogReader catalogReader; + protected final RelOptCluster cluster; + private DefaultValueFactory defaultValueFactory; + private SubqueryConverter subqueryConverter; + protected final List leaves = new ArrayList(); + private final List dynamicParamSqlNodes = + new ArrayList(); + private final SqlOperatorTable opTab; + private boolean shouldConvertTableAccess; + protected final RelDataTypeFactory typeFactory; + private final SqlNodeToRexConverter exprConverter; + private boolean decorrelationEnabled; + private boolean trimUnusedFields; + private boolean shouldCreateValuesRel; + private boolean isExplain; + private int nDynamicParamsInExplain; + + /** + * Fields used in name resolution for correlated subqueries. + */ + private final Map mapCorrelToDeferred = + new HashMap(); + private int nextCorrel = 0; + + private static final String CORREL_PREFIX = "$cor"; + + /** + * Stack of names of datasets requested by the + * TABLE(SAMPLE(<datasetName>, <query>)) construct. + */ + private final Stack datasetStack = new Stack(); + + /** + * Mapping of non-correlated subqueries that have been converted to their + * equivalent constants. Used to avoid re-evaluating the subquery if it's + * already been evaluated. + */ + private final Map mapConvertedNonCorrSubqs = + new HashMap(); + + public final RelOptTable.ViewExpander viewExpander; + + //~ Constructors ----------------------------------------------------------- + /** + * Creates a converter. + * + * @param viewExpander Preparing statement + * @param validator Validator + * @param catalogReader Schema + * @param planner Planner + * @param rexBuilder Rex builder + * @param convertletTable Expression converter + */ + public SqlToRelConverter( + RelOptTable.ViewExpander viewExpander, + SqlValidator validator, + Prepare.CatalogReader catalogReader, + RelOptPlanner planner, + RexBuilder rexBuilder, + SqlRexConvertletTable convertletTable) { + this.viewExpander = viewExpander; + this.opTab = + (validator + == null) ? SqlStdOperatorTable.instance() + : validator.getOperatorTable(); + this.validator = validator; + this.catalogReader = catalogReader; + this.defaultValueFactory = new NullDefaultValueFactory(); + this.subqueryConverter = new NoOpSubqueryConverter(); + this.rexBuilder = rexBuilder; + this.typeFactory = rexBuilder.getTypeFactory(); + RelOptQuery query = new RelOptQuery(planner); + this.cluster = query.createCluster(typeFactory, rexBuilder); + this.shouldConvertTableAccess = true; + this.exprConverter = + new SqlNodeToRexConverterImpl(convertletTable); + decorrelationEnabled = true; + trimUnusedFields = false; + shouldCreateValuesRel = true; + isExplain = false; + nDynamicParamsInExplain = 0; + } + + //~ Methods ---------------------------------------------------------------- + + /** + * @return the RelOptCluster in use. + */ + public RelOptCluster getCluster() { + return cluster; + } + + /** + * Returns the row-expression builder. + */ + public RexBuilder getRexBuilder() { + return rexBuilder; + } + + /** + * Returns the number of dynamic parameters encountered during translation; + * this must only be called after {@link #convertQuery}. + * + * @return number of dynamic parameters + */ + public int getDynamicParamCount() { + return dynamicParamSqlNodes.size(); + } + + /** + * Returns the type inferred for a dynamic parameter. + * + * @param index 0-based index of dynamic parameter + * @return inferred type, never null + */ + public RelDataType getDynamicParamType(int index) { + SqlNode sqlNode = dynamicParamSqlNodes.get(index); + if (sqlNode == null) { + throw Util.needToImplement("dynamic param type inference"); + } + return validator.getValidatedNodeType(sqlNode); + } + + /** + * Returns the current count of the number of dynamic parameters in an + * EXPLAIN PLAN statement. + * + * @param increment if true, increment the count + * @return the current count before the optional increment + */ + public int getDynamicParamCountInExplain(boolean increment) { + int retVal = nDynamicParamsInExplain; + if (increment) { + ++nDynamicParamsInExplain; + } + return retVal; + } + + /** + * @return mapping of non-correlated subqueries that have been converted to + * the constants that they evaluate to + */ + public Map getMapConvertedNonCorrSubqs() { + return mapConvertedNonCorrSubqs; + } + + /** + * Adds to the current map of non-correlated converted subqueries the + * elements from another map that contains non-correlated subqueries that + * have been converted by another SqlToRelConverter. + * + * @param alreadyConvertedNonCorrSubqs the other map + */ + public void addConvertedNonCorrSubqs( + Map alreadyConvertedNonCorrSubqs) { + mapConvertedNonCorrSubqs.putAll(alreadyConvertedNonCorrSubqs); + } + + /** + * Set a new DefaultValueFactory. To have any effect, this must be called + * before any convert method. + * + * @param factory new DefaultValueFactory + */ + public void setDefaultValueFactory(DefaultValueFactory factory) { + defaultValueFactory = factory; + } + + /** + * Sets a new SubqueryConverter. To have any effect, this must be called + * before any convert method. + * + * @param converter new SubqueryConverter + */ + public void setSubqueryConverter(SubqueryConverter converter) { + subqueryConverter = converter; + } + + /** + * Indicates that the current statement is part of an EXPLAIN PLAN statement + * + * @param nDynamicParams number of dynamic parameters in the statement + */ + public void setIsExplain(int nDynamicParams) { + isExplain = true; + nDynamicParamsInExplain = nDynamicParams; + } + + /** + * Controls whether table access references are converted to physical rels + * immediately. The optimizer doesn't like leaf rels to have + * {@link Convention#NONE}. However, if we are doing further conversion + * passes (e.g. {@link RelStructuredTypeFlattener}), then we may need to + * defer conversion. To have any effect, this must be called before any + * convert method. + * + * @param enabled true for immediate conversion (the default); false to + * generate logical TableAccessRel instances + */ + public void enableTableAccessConversion(boolean enabled) { + shouldConvertTableAccess = enabled; + } + + /** + * Controls whether instances of {@link ValuesRel} are generated. These may + * not be supported by all physical implementations. To have any effect, + * this must be called before any convert method. + * + * @param enabled true to allow ValuesRel to be generated (the default); + * false to force substitution of ProjectRel+OneRowRel instead + */ + public void enableValuesRelCreation(boolean enabled) { + shouldCreateValuesRel = enabled; + } + + private void checkConvertedType(SqlNode query, RelNode result) { + if (!query.isA(SqlKind.DML)) { + // Verify that conversion from SQL to relational algebra did + // not perturb any type information. (We can't do this if the + // SQL statement is something like an INSERT which has no + // validator type information associated with its result, + // hence the namespace check above.) + RelDataType convertedRowType = result.getRowType(); + if (!checkConvertedRowType(query, convertedRowType)) { + RelDataType validatedRowType = + validator.getValidatedNodeType(query); + validatedRowType = uniquifyFields(validatedRowType); + throw Util.newInternal( + "Conversion to relational algebra failed to preserve " + + "datatypes:\n" + + "validated type:\n" + + validatedRowType.getFullTypeString() + + "\nconverted type:\n" + + convertedRowType.getFullTypeString() + + "\nrel:\n" + + RelOptUtil.toString(result)); + } + } + } + + public RelNode flattenTypes( + RelNode rootRel, + boolean restructure) { + RelStructuredTypeFlattener typeFlattener = + new RelStructuredTypeFlattener(rexBuilder, createToRelContext()); + return typeFlattener.rewrite(rootRel, restructure); + } + + /** + * If subquery is correlated and decorrelation is enabled, performs + * decorrelation. + * + * @param query Query + * @param rootRel Root relational expression + * @return New root relational expression after decorrelation + */ + public RelNode decorrelate(SqlNode query, RelNode rootRel) { + if (!enableDecorrelation()) { + return rootRel; + } + final RelNode result = decorrelateQuery(rootRel); + if (result != rootRel) { + checkConvertedType(query, result); + } + return result; + } + + /** + * Walks over a tree of relational expressions, replacing each + * {@link RelNode} with a 'slimmed down' relational expression that projects + * only the fields required by its consumer. + * + *

This may make things easier for the optimizer, by removing crud that + * would expand the search space, but is difficult for the optimizer itself + * to do it, because optimizer rules must preserve the number and type of + * fields. Hence, this transform that operates on the entire tree, similar + * to the {@link RelStructuredTypeFlattener type-flattening transform}. + * + *

Currently this functionality is disabled in farrago/luciddb; the + * default implementation of this method does nothing. + * + * @param rootRel Relational expression that is at the root of the tree + * @return Trimmed relational expression + */ + public RelNode trimUnusedFields(RelNode rootRel) { + // Trim fields that are not used by their consumer. + if (isTrimUnusedFields()) { + final RelFieldTrimmer trimmer = newFieldTrimmer(); + rootRel = trimmer.trim(rootRel); + boolean dumpPlan = SQL2REL_LOGGER.isLoggable(Level.FINE); + if (dumpPlan) { + SQL2REL_LOGGER.fine( + RelOptUtil.dumpPlan( + "Plan after trimming unused fields", + rootRel, + false, + SqlExplainLevel.EXPPLAN_ATTRIBUTES)); + } + } + return rootRel; + } + + /** + * Creates a RelFieldTrimmer. + * + * @return Field trimmer + */ + protected RelFieldTrimmer newFieldTrimmer() { + return new RelFieldTrimmer(validator); + } + + /** + * Converts an unvalidated query's parse tree into a relational expression. + * + * @param query Query to convert + * @param needsValidation Whether to validate the query before converting; + * false if the query has already been + * validated. + * @param top Whether the query is top-level, say if its result + * will become a JDBC result set; false if + * the query will be part of a view. + */ + public RelNode convertQuery( + SqlNode query, + final boolean needsValidation, + final boolean top) { + if (needsValidation) { + query = validator.validate(query); + } + + RelNode result = convertQueryRecursive(query, top, null); + checkConvertedType(query, result); + + boolean dumpPlan = SQL2REL_LOGGER.isLoggable(Level.FINE); + if (dumpPlan) { + SQL2REL_LOGGER.fine( + RelOptUtil.dumpPlan( + "Plan after converting SqlNode to RelNode", + result, + false, + SqlExplainLevel.EXPPLAN_ATTRIBUTES)); + } + + return result; + } + + protected boolean checkConvertedRowType( + SqlNode query, + RelDataType convertedRowType) { + RelDataType validatedRowType = validator.getValidatedNodeType(query); + validatedRowType = uniquifyFields(validatedRowType); + + return RelOptUtil.equal( + "validated row type", + validatedRowType, + "converted row type", + convertedRowType, + false); + } + + protected RelDataType uniquifyFields(RelDataType rowType) { + return validator.getTypeFactory().createStructType( + RelOptUtil.getFieldTypeList(rowType), + SqlValidatorUtil.uniquify(rowType.getFieldNames())); + } + + /** + * Converts a SELECT statement's parse tree into a relational expression. + */ + public RelNode convertSelect(SqlSelect select) { + final SqlValidatorScope selectScope = validator.getWhereScope(select); + final Blackboard bb = createBlackboard(selectScope, null); + convertSelectImpl(bb, select); + return bb.root; + } + + /** + * Factory method for creating translation workspace. + */ + protected Blackboard createBlackboard( + SqlValidatorScope scope, + Map nameToNodeMap) { + return new Blackboard(scope, nameToNodeMap); + } + + /** + * Implementation of {@link #convertSelect(SqlSelect)}; derived class may + * override. + */ + protected void convertSelectImpl( + final Blackboard bb, + SqlSelect select) { + convertFrom( + bb, + select.getFrom()); + convertWhere( + bb, + select.getWhere()); + + List orderExprList = new ArrayList(); + List collationList = + new ArrayList(); + gatherOrderExprs( + bb, + select, + select.getOrderList(), + orderExprList, + collationList); + final RelCollation collation = + cluster.traitSetOf().canonize(RelCollationImpl.of(collationList)); + + if (validator.isAggregate(select)) { + convertAgg( + bb, + select, + orderExprList); + } else { + convertSelectList( + bb, + select, + orderExprList); + } + + if (select.isDistinct()) { + distinctify(bb, true); + } + convertOrder( + select, bb, collation, orderExprList, select.getOffset(), + select.getFetch()); + bb.setRoot(bb.root, true); + } + + /** + * Having translated 'SELECT ... FROM ... [GROUP BY ...] [HAVING ...]', adds + * a relational expression to make the results unique. + * + *

If the SELECT clause contains duplicate expressions, adds {@link + * ProjectRel}s so that we are grouping on the minimal set of keys. The + * performance gain isn't huge, but it is difficult to detect these + * duplicate expressions later. + * + * @param bb Blackboard + * @param checkForDupExprs Check for duplicate expressions + */ + private void distinctify( + Blackboard bb, + boolean checkForDupExprs) { + // Look for duplicate expressions in the project. + // Say we have 'select x, y, x, z'. + // Then dups will be {[2, 0]} + // and oldToNew will be {[0, 0], [1, 1], [2, 0], [3, 2]} + RelNode rel = bb.root; + if (checkForDupExprs && (rel instanceof ProjectRel)) { + ProjectRel project = (ProjectRel) rel; + final List projectExprs = project.getProjects(); + List origins = new ArrayList(); + int dupCount = 0; + for (int i = 0; i < projectExprs.size(); i++) { + int x = findExpr(projectExprs.get(i), projectExprs, i); + if (x >= 0) { + origins.add(x); + ++dupCount; + } else { + origins.add(i); + } + } + if (dupCount == 0) { + distinctify(bb, false); + return; + } + + final Map squished = Maps.newHashMap(); + final List fields = rel.getRowType().getFieldList(); + final List> newProjects = Lists.newArrayList(); + for (int i = 0; i < fields.size(); i++) { + if (origins.get(i) == i) { + squished.put(i, newProjects.size()); + newProjects.add(RexInputRef.of2(i, fields)); + } + } + rel = + new ProjectRel( + cluster, + rel, + Pair.left(newProjects), + Pair.right(newProjects), + ProjectRel.Flags.BOXED); + + bb.root = rel; + distinctify(bb, false); + rel = bb.root; + + // Create the expressions to reverse the mapping. + // Project($0, $1, $0, $2). + final List> undoProjects = Lists.newArrayList(); + for (int i = 0; i < fields.size(); i++) { + final int origin = origins.get(i); + RelDataTypeField field = fields.get(i); + undoProjects.add( + Pair.of( + (RexNode) new RexInputRef( + squished.get(origin), field.getType()), + field.getName())); + } + + rel = + new ProjectRel( + cluster, + rel, + Pair.left(undoProjects), + Pair.right(undoProjects), + ProjectRel.Flags.BOXED); + + bb.setRoot( + rel, + false); + + return; + } + + // Usual case: all of the expressions in the SELECT clause are + // different. + rel = + createAggregate( + bb, + BitSets.range(rel.getRowType().getFieldCount()), + ImmutableList.of()); + + bb.setRoot( + rel, + false); + } + + private int findExpr(RexNode seek, List exprs, int count) { + for (int i = 0; i < count; i++) { + RexNode expr = exprs.get(i); + if (expr.toString().equals(seek.toString())) { + return i; + } + } + return -1; + } + + /** + * Converts a query's ORDER BY clause, if any. + * + * @param select Query + * @param bb Blackboard + * @param collation Collation list + * @param orderExprList Method populates this list with orderBy expressions + * not present in selectList + * @param offset Expression for number of rows to discard before + * returning first row + * @param fetch Expression for number of rows to fetch + */ + protected void convertOrder( + SqlSelect select, + Blackboard bb, + RelCollation collation, + List orderExprList, + SqlNode offset, + SqlNode fetch) { + if (select.getOrderList() == null) { + assert collation.getFieldCollations().isEmpty(); + if (offset == null && fetch == null) { + return; + } + } + + // Create a sorter using the previously constructed collations. + bb.setRoot( + new SortRel( + cluster, + cluster.traitSetOf(Convention.NONE, collation), + bb.root, + collation, + offset == null ? null : convertExpression(offset), + fetch == null ? null : convertExpression(fetch)), + false); + + // If extra expressions were added to the project list for sorting, + // add another project to remove them. + if (orderExprList.size() > 0) { + List exprs = new ArrayList(); + final RelDataType rowType = bb.root.getRowType(); + final int fieldCount = + rowType.getFieldCount() - orderExprList.size(); + for (int i = 0; i < fieldCount; i++) { + exprs.add(rexBuilder.makeInputRef(bb.root, i)); + } + bb.setRoot( + new ProjectRel( + cluster, + cluster.traitSetOf(RelCollationImpl.PRESERVE), + bb.root, + exprs, + cluster.getTypeFactory().createStructType( + rowType.getFieldList().subList(0, fieldCount)), + ProjectRelBase.Flags.BOXED), + false); + } + } + + /** + * Returns whether a given node contains a {@link SqlInOperator}. + * + * @param node a RexNode tree + */ + private static boolean containsInOperator( + SqlNode node) { + try { + SqlVisitor visitor = + new SqlBasicVisitor() { + public Void visit(SqlCall call) { + if (call.getOperator() instanceof SqlInOperator) { + throw new Util.FoundOne(call); + } + return super.visit(call); + } + }; + node.accept(visitor); + return false; + } catch (Util.FoundOne e) { + Util.swallow(e, null); + return true; + } + } + + /** + * Push down all the NOT logical operators into any IN/NOT IN operators. + * + * @param sqlNode the root node from which to look for NOT operators + * @return the transformed SqlNode representation with NOT pushed down. + */ + private static SqlNode pushDownNotForIn(SqlNode sqlNode) { + if ((sqlNode instanceof SqlCall) && containsInOperator(sqlNode)) { + SqlCall sqlCall = (SqlCall) sqlNode; + if ((sqlCall.getOperator() == SqlStdOperatorTable.AND) + || (sqlCall.getOperator() == SqlStdOperatorTable.OR)) { + SqlNode[] sqlOperands = ((SqlBasicCall) sqlCall).operands; + for (int i = 0; i < sqlOperands.length; i++) { + sqlOperands[i] = pushDownNotForIn(sqlOperands[i]); + } + return sqlNode; + } else if (sqlCall.getOperator() == SqlStdOperatorTable.NOT) { + SqlNode childNode = sqlCall.operand(0); + assert childNode instanceof SqlCall; + SqlBasicCall childSqlCall = (SqlBasicCall) childNode; + if (childSqlCall.getOperator() == SqlStdOperatorTable.AND) { + SqlNode[] andOperands = childSqlCall.getOperands(); + SqlNode[] orOperands = new SqlNode[andOperands.length]; + for (int i = 0; i < orOperands.length; i++) { + orOperands[i] = + SqlStdOperatorTable.NOT.createCall( + SqlParserPos.ZERO, + andOperands[i]); + } + for (int i = 0; i < orOperands.length; i++) { + orOperands[i] = pushDownNotForIn(orOperands[i]); + } + return SqlStdOperatorTable.OR.createCall(SqlParserPos.ZERO, + orOperands[0], orOperands[1]); + } else if (childSqlCall.getOperator() == SqlStdOperatorTable.OR) { + SqlNode[] orOperands = childSqlCall.getOperands(); + SqlNode[] andOperands = new SqlNode[orOperands.length]; + for (int i = 0; i < andOperands.length; i++) { + andOperands[i] = + SqlStdOperatorTable.NOT.createCall( + SqlParserPos.ZERO, + orOperands[i]); + } + for (int i = 0; i < andOperands.length; i++) { + andOperands[i] = pushDownNotForIn(andOperands[i]); + } + return SqlStdOperatorTable.AND.createCall(SqlParserPos.ZERO, + andOperands[0], andOperands[1]); + } else if (childSqlCall.getOperator() == SqlStdOperatorTable.NOT) { + SqlNode[] notOperands = childSqlCall.getOperands(); + assert notOperands.length == 1; + return pushDownNotForIn(notOperands[0]); + } else if (childSqlCall.getOperator() instanceof SqlInOperator) { + SqlNode[] inOperands = childSqlCall.getOperands(); + SqlInOperator inOp = + (SqlInOperator) childSqlCall.getOperator(); + if (inOp.isNotIn()) { + return SqlStdOperatorTable.IN.createCall( + SqlParserPos.ZERO, + inOperands[0], + inOperands[1]); + } else { + return SqlStdOperatorTable.NOT_IN.createCall( + SqlParserPos.ZERO, + inOperands[0], + inOperands[1]); + } + } else { + // childSqlCall is "leaf" node in a logical expression tree + // (only considering AND, OR, NOT) + return sqlNode; + } + } else { + // sqlNode is "leaf" node in a logical expression tree + // (only considering AND, OR, NOT) + return sqlNode; + } + } else { + // tree rooted at sqlNode does not contain inOperator + return sqlNode; + } + } + + /** + * Converts a WHERE clause. + * + * @param bb Blackboard + * @param where WHERE clause, may be null + */ + private void convertWhere( + final Blackboard bb, + final SqlNode where) { + if (where == null) { + return; + } + SqlNode newWhere = pushDownNotForIn(where); + replaceSubqueries(bb, newWhere, RelOptUtil.Logic.UNKNOWN_AS_FALSE); + final RexNode convertedWhere = bb.convertExpression(newWhere); + + // only allocate filter if the condition is not TRUE + if (!convertedWhere.isAlwaysTrue()) { + bb.setRoot( + RelOptUtil.createFilter(bb.root, convertedWhere), + false); + } + } + + private void replaceSubqueries( + final Blackboard bb, + final SqlNode expr, + RelOptUtil.Logic logic) { + findSubqueries(bb, expr, logic, false); + for (SubQuery node : bb.subqueryList) { + substituteSubquery(bb, node); + } + } + + private void substituteSubquery(Blackboard bb, SubQuery subQuery) { + final RexNode expr = subQuery.expr; + if (expr != null) { + // Already done. + return; + } + + final SqlBasicCall call; + final RelNode rel; + final SqlNode query; + final Pair converted; + switch (subQuery.node.getKind()) { + case CURSOR: + convertCursor(bb, subQuery); + return; + + case MULTISET_QUERY_CONSTRUCTOR: + case MULTISET_VALUE_CONSTRUCTOR: + rel = convertMultisets(ImmutableList.of(subQuery.node), bb); + subQuery.expr = bb.register(rel, JoinRelType.INNER); + return; + + case IN: + call = (SqlBasicCall) subQuery.node; + final SqlNode[] operands = call.getOperands(); + + SqlNode leftKeyNode = operands[0]; + query = operands[1]; + + final List leftKeys; + switch (leftKeyNode.getKind()) { + case ROW: + leftKeys = Lists.newArrayList(); + for (SqlNode sqlExpr : ((SqlBasicCall) leftKeyNode).getOperandList()) { + leftKeys.add(bb.convertExpression(sqlExpr)); + } + break; + default: + leftKeys = ImmutableList.of(bb.convertExpression(leftKeyNode)); + } + + final boolean isNotIn = ((SqlInOperator) call.getOperator()).isNotIn(); + if (query instanceof SqlNodeList) { + SqlNodeList valueList = (SqlNodeList) query; + if (!containsNullLiteral(valueList) + && valueList.size() < getInSubqueryThreshold()) { + // We're under the threshold, so convert to OR. + subQuery.expr = + convertInToOr( + bb, + leftKeys, + valueList, + isNotIn); + return; + } + + // Otherwise, let convertExists translate + // values list into an inline table for the + // reference to Q below. + } + + // Project out the search columns from the left side + + // Q1: + // "select from emp where emp.deptno in (select col1 from T)" + // + // is converted to + // + // "select from + // emp inner join (select distinct col1 from T)) q + // on emp.deptno = q.col1 + // + // Q2: + // "select from emp where emp.deptno not in (Q)" + // + // is converted to + // + // "select from + // emp left outer join (select distinct col1, TRUE from T) q + // on emp.deptno = q.col1 + // where emp.deptno <> null + // and q.indicator <> TRUE" + // + final boolean outerJoin = bb.subqueryNeedsOuterJoin + || isNotIn + || subQuery.logic == RelOptUtil.Logic.TRUE_FALSE_UNKNOWN; + converted = + convertExists(query, RelOptUtil.SubqueryType.IN, subQuery.logic, + outerJoin); + if (converted.right) { + // Generate + // emp CROSS JOIN (SELECT COUNT(*) AS c, + // COUNT(deptno) AS ck FROM dept) + final RelDataType longType = + typeFactory.createSqlType(SqlTypeName.BIGINT); + final RelNode seek = converted.left.getInput(0); // fragile + final int keyCount = leftKeys.size(); + final List args = ImmutableIntList.range(0, keyCount); + AggregateRel aggregate = + new AggregateRel(cluster, seek, BitSets.of(), + ImmutableList.of( + new AggregateCall(SqlStdOperatorTable.COUNT, false, + ImmutableList.of(), longType, null), + new AggregateCall(SqlStdOperatorTable.COUNT, false, + args, longType, null))); + JoinRel join = + new JoinRel(cluster, bb.root, aggregate, + rexBuilder.makeLiteral(true), JoinRelType.INNER, + ImmutableSet.of()); + bb.setRoot(join, false); + } + RexNode rex = + bb.register(converted.left, + outerJoin ? JoinRelType.LEFT : JoinRelType.INNER, leftKeys); + + subQuery.expr = translateIn(subQuery, bb.root, rex); + if (isNotIn) { + subQuery.expr = + rexBuilder.makeCall(SqlStdOperatorTable.NOT, subQuery.expr); + } + return; + + case EXISTS: + // "select from emp where exists (select a from T)" + // + // is converted to the following if the subquery is correlated: + // + // "select from emp left outer join (select AGG_TRUE() as indicator + // from T group by corr_var) q where q.indicator is true" + // + // If there is no correlation, the expression is replaced with a + // boolean indicating whether the subquery returned 0 or >= 1 row. + call = (SqlBasicCall) subQuery.node; + query = call.getOperands()[0]; + converted = convertExists(query, RelOptUtil.SubqueryType.EXISTS, + subQuery.logic, true); + assert !converted.right; + if (convertNonCorrelatedSubQuery(subQuery, bb, converted.left, true)) { + return; + } + subQuery.expr = bb.register(converted.left, JoinRelType.LEFT); + return; + + case SCALAR_QUERY: + // Convert the subquery. If it's non-correlated, convert it + // to a constant expression. + call = (SqlBasicCall) subQuery.node; + query = call.getOperands()[0]; + converted = convertExists(query, RelOptUtil.SubqueryType.SCALAR, + subQuery.logic, true); + assert !converted.right; + if (convertNonCorrelatedSubQuery(subQuery, bb, converted.left, false)) { + return; + } + rel = convertToSingleValueSubq(query, converted.left); + subQuery.expr = bb.register(rel, JoinRelType.LEFT); + return; + + case SELECT: + // This is used when converting multiset queries: + // + // select * from unnest(select multiset[deptno] from emps); + // + converted = convertExists(subQuery.node, RelOptUtil.SubqueryType.SCALAR, + subQuery.logic, true); + assert !converted.right; + subQuery.expr = bb.register(converted.left, JoinRelType.LEFT); + return; + + default: + throw Util.newInternal("unexpected kind of subquery :" + subQuery.node); + } + } + + private RexNode translateIn(SubQuery subQuery, RelNode root, + final RexNode rex) { + switch (subQuery.logic) { + case TRUE: + return rexBuilder.makeLiteral(true); + + case UNKNOWN_AS_FALSE: + assert rex instanceof RexRangeRef; + final int fieldCount = rex.getType().getFieldCount(); + RexNode rexNode = rexBuilder.makeFieldAccess(rex, fieldCount - 1); + rexNode = rexBuilder.makeCall(SqlStdOperatorTable.IS_TRUE, rexNode); + + // Then append the IS NOT NULL(leftKeysForIn). + // + // RexRangeRef contains the following fields: + // leftKeysForIn, + // rightKeysForIn (the original subquery select list), + // nullIndicator + // + // The first two lists contain the same number of fields. + final int k = (fieldCount - 1) / 2; + for (int i = 0; i < k; i++) { + rexNode = + rexBuilder.makeCall( + SqlStdOperatorTable.AND, + rexNode, + rexBuilder.makeCall( + SqlStdOperatorTable.IS_NOT_NULL, + rexBuilder.makeFieldAccess(rex, i))); + } + return rexNode; + + case TRUE_FALSE_UNKNOWN: + case UNKNOWN_AS_TRUE: + // select e.deptno, + // case + // when ct.c = 0 then false + // when dt.i is not null then true + // when e.deptno is null then null + // when ct.ck < ct.c then null + // else false + // end + // from e + // cross join (select count(*) as c, count(deptno) as ck from v) as ct + // left join (select distinct deptno, true as i from v) as dt + // on e.deptno = dt.deptno + final JoinRelBase join = (JoinRelBase) root; + final ProjectRelBase left = (ProjectRelBase) join.getLeft(); + final RelNode leftLeft = ((JoinRelBase) left.getInput(0)).getLeft(); + final int leftLeftCount = leftLeft.getRowType().getFieldCount(); + final RelDataType nullableBooleanType = + typeFactory.createTypeWithNullability( + typeFactory.createSqlType(SqlTypeName.BOOLEAN), true); + final RelDataType longType = + typeFactory.createSqlType(SqlTypeName.BIGINT); + final RexNode cRef = rexBuilder.makeInputRef(root, leftLeftCount); + final RexNode ckRef = rexBuilder.makeInputRef(root, leftLeftCount + 1); + final RexNode iRef = + rexBuilder.makeInputRef(root, root.getRowType().getFieldCount() - 1); + + final RexLiteral zero = + rexBuilder.makeExactLiteral(BigDecimal.ZERO, longType); + final RexLiteral trueLiteral = rexBuilder.makeLiteral(true); + final RexLiteral falseLiteral = rexBuilder.makeLiteral(false); + final RexNode unknownLiteral = + rexBuilder.makeNullLiteral(SqlTypeName.BOOLEAN); + + final ImmutableList.Builder args = ImmutableList.builder(); + args.add(rexBuilder.makeCall(SqlStdOperatorTable.EQUALS, cRef, zero), + falseLiteral, + rexBuilder.makeCall(SqlStdOperatorTable.IS_NOT_NULL, iRef), + trueLiteral); + final JoinInfo joinInfo = join.analyzeCondition(); + for (int leftKey : joinInfo.leftKeys) { + final RexNode kRef = rexBuilder.makeInputRef(root, leftKey); + args.add(rexBuilder.makeCall(SqlStdOperatorTable.IS_NULL, kRef), + unknownLiteral); + } + args.add(rexBuilder.makeCall(SqlStdOperatorTable.LESS_THAN, ckRef, cRef), + unknownLiteral, + falseLiteral); + + return rexBuilder.makeCall( + nullableBooleanType, + SqlStdOperatorTable.CASE, + args.build()); + + default: + throw new AssertionError(subQuery.logic); + } + } + + private static boolean containsNullLiteral(SqlNodeList valueList) { + for (SqlNode node : valueList.getList()) { + if (node instanceof SqlLiteral) { + SqlLiteral lit = (SqlLiteral) node; + if (lit.getValue() == null) { + return true; + } + } + } + return false; + } + + /** + * Determines if a subquery is non-correlated and if so, converts it to a + * constant. + * + * @param subQuery the call that references the subquery + * @param bb blackboard used to convert the subquery + * @param converted RelNode tree corresponding to the subquery + * @param isExists true if the subquery is part of an EXISTS expression + * @return if the subquery can be converted to a constant + */ + private boolean convertNonCorrelatedSubQuery( + SubQuery subQuery, + Blackboard bb, + RelNode converted, + boolean isExists) { + SqlCall call = (SqlBasicCall) subQuery.node; + if (subqueryConverter.canConvertSubquery() + && isSubqNonCorrelated(converted, bb)) { + // First check if the subquery has already been converted + // because it's a nested subquery. If so, don't re-evaluate + // it again. + RexNode constExpr = mapConvertedNonCorrSubqs.get(call); + if (constExpr == null) { + constExpr = + subqueryConverter.convertSubquery( + call, + this, + isExists, + isExplain); + } + if (constExpr != null) { + subQuery.expr = constExpr; + mapConvertedNonCorrSubqs.put(call, constExpr); + return true; + } + } + return false; + } + + /** + * Converts the RelNode tree for a select statement to a select that + * produces a single value. + * + * @param query the query + * @param plan the original RelNode tree corresponding to the statement + * @return the converted RelNode tree + */ + public RelNode convertToSingleValueSubq( + SqlNode query, + RelNode plan) { + // Check whether query is guaranteed to produce a single value. + if (query instanceof SqlSelect) { + SqlSelect select = (SqlSelect) query; + SqlNodeList selectList = select.getSelectList(); + SqlNodeList groupList = select.getGroup(); + + if ((selectList.size() == 1) + && ((groupList == null) || (groupList.size() == 0))) { + SqlNode selectExpr = selectList.get(0); + if (selectExpr instanceof SqlCall) { + SqlCall selectExprCall = (SqlCall) selectExpr; + if (selectExprCall.getOperator() + instanceof SqlAggFunction) { + return plan; + } + } + } + } + + // If not, project SingleValueAgg + return RelOptUtil.createSingleValueAggRel( + cluster, + plan); + } + + /** + * Converts "x IN (1, 2, ...)" to "x=1 OR x=2 OR ...". + * + * @param leftKeys LHS + * @param valuesList RHS + * @param isNotIn is this a NOT IN operator + * @return converted expression + */ + private RexNode convertInToOr( + final Blackboard bb, + final List leftKeys, + SqlNodeList valuesList, + boolean isNotIn) { + List comparisons = new ArrayList(); + for (SqlNode rightVals : valuesList) { + RexNode rexComparison; + if (leftKeys.size() == 1) { + rexComparison = + rexBuilder.makeCall( + SqlStdOperatorTable.EQUALS, + leftKeys.get(0), + bb.convertExpression(rightVals)); + } else { + assert rightVals instanceof SqlCall; + final SqlBasicCall call = (SqlBasicCall) rightVals; + assert (call.getOperator() instanceof SqlRowOperator) + && call.getOperands().length == leftKeys.size(); + rexComparison = + RexUtil.composeConjunction( + rexBuilder, + Iterables.transform( + Pair.zip(leftKeys, call.getOperandList()), + new Function, RexNode>() { + public RexNode apply(Pair pair) { + return rexBuilder.makeCall(SqlStdOperatorTable.EQUALS, + pair.left, bb.convertExpression(pair.right)); + } + }), + false); + } + comparisons.add(rexComparison); + } + + RexNode result = + RexUtil.composeDisjunction(rexBuilder, comparisons, true); + assert result != null; + + if (isNotIn) { + result = + rexBuilder.makeCall( + SqlStdOperatorTable.NOT, + result); + } + + return result; + } + + /** + * Gets the list size threshold under which {@link #convertInToOr} is used. + * Lists of this size or greater will instead be converted to use a join + * against an inline table ({@link ValuesRel}) rather than a predicate. A + * threshold of 0 forces usage of an inline table in all cases; a threshold + * of Integer.MAX_VALUE forces usage of OR in all cases + * + * @return threshold, default 20 + */ + protected int getInSubqueryThreshold() { + // OVERRIDE POINT + return Integer.MAX_VALUE; // was 20 + } + + /** + * Converts an EXISTS or IN predicate into a join. For EXISTS, the subquery + * produces an indicator variable, and the result is a relational expression + * which outer joins that indicator to the original query. After performing + * the outer join, the condition will be TRUE if the EXISTS condition holds, + * NULL otherwise. + * + * @param seek A query, for example 'select * from emp' or + * 'values (1,2,3)' or '('Foo', 34)'. + * @param subqueryType Whether sub-query is IN, EXISTS or scalar + * @param logic Whether the answer needs to be in full 3-valued logic (TRUE, + * FALSE, UNKNOWN) will be required, or whether we can accept an + * approximation (say representing UNKNOWN as FALSE) + * @param needsOuterJoin Whether an outer join is needed + * @return join expression + * @pre extraExpr == null || extraName != null + */ + private Pair convertExists( + SqlNode seek, + RelOptUtil.SubqueryType subqueryType, + RelOptUtil.Logic logic, + boolean needsOuterJoin) { + final SqlValidatorScope seekScope = + (seek instanceof SqlSelect) + ? validator.getSelectScope((SqlSelect) seek) + : null; + final Blackboard seekBb = createBlackboard(seekScope, null); + RelNode seekRel = convertQueryOrInList(seekBb, seek); + + return RelOptUtil.createExistsPlan(seekRel, subqueryType, logic, + needsOuterJoin); + } + + private RelNode convertQueryOrInList( + Blackboard bb, + SqlNode seek) { + // NOTE: Once we start accepting single-row queries as row constructors, + // there will be an ambiguity here for a case like X IN ((SELECT Y FROM + // Z)). The SQL standard resolves the ambiguity by saying that a lone + // select should be interpreted as a table expression, not a row + // expression. The semantic difference is that a table expression can + // return multiple rows. + if (seek instanceof SqlNodeList) { + return convertRowValues( + bb, + seek, + ((SqlNodeList) seek).getList(), + false, + null); + } else { + return convertQueryRecursive(seek, false, null); + } + } + + private RelNode convertRowValues( + Blackboard bb, + SqlNode rowList, + Collection rows, + boolean allowLiteralsOnly, + RelDataType targetRowType) { + // NOTE jvs 30-Apr-2006: We combine all rows consisting entirely of + // literals into a single ValuesRel; this gives the optimizer a smaller + // input tree. For everything else (computed expressions, row + // subqueries), we union each row in as a projection on top of a + // OneRowRel. + + final List> tupleList = + new ArrayList>(); + final RelDataType rowType; + if (targetRowType != null) { + rowType = targetRowType; + } else { + rowType = + SqlTypeUtil.promoteToRowType( + typeFactory, + validator.getValidatedNodeType(rowList), + null); + } + + List unionInputs = new ArrayList(); + for (SqlNode node : rows) { + SqlBasicCall call; + if (isRowConstructor(node)) { + call = (SqlBasicCall) node; + List tuple = new ArrayList(); + for (SqlNode operand : call.operands) { + RexLiteral rexLiteral = + convertLiteralInValuesList( + operand, + bb, + rowType, + tuple.size()); + if ((rexLiteral == null) && allowLiteralsOnly) { + return null; + } + if ((rexLiteral == null) || !shouldCreateValuesRel) { + // fallback to convertRowConstructor + tuple = null; + break; + } + tuple.add(rexLiteral); + } + if (tuple != null) { + tupleList.add(tuple); + continue; + } + } else { + RexLiteral rexLiteral = + convertLiteralInValuesList( + node, + bb, + rowType, + 0); + if ((rexLiteral != null) && shouldCreateValuesRel) { + tupleList.add( + Collections.singletonList(rexLiteral)); + continue; + } else { + if ((rexLiteral == null) && allowLiteralsOnly) { + return null; + } + } + + // convert "1" to "row(1)" + call = + (SqlBasicCall) SqlStdOperatorTable.ROW.createCall( + SqlParserPos.ZERO, + node); + } + unionInputs.add(convertRowConstructor(bb, call)); + } + ValuesRel valuesRel = + new ValuesRel( + cluster, + rowType, + tupleList); + RelNode resultRel; + if (unionInputs.isEmpty()) { + resultRel = valuesRel; + } else { + if (!tupleList.isEmpty()) { + unionInputs.add(valuesRel); + } + UnionRel unionRel = + new UnionRel( + cluster, + unionInputs, + true); + resultRel = unionRel; + } + leaves.add(resultRel); + return resultRel; + } + + private RexLiteral convertLiteralInValuesList( + SqlNode sqlNode, + Blackboard bb, + RelDataType rowType, + int iField) { + if (!(sqlNode instanceof SqlLiteral)) { + return null; + } + RelDataTypeField field = rowType.getFieldList().get(iField); + RelDataType type = field.getType(); + if (type.isStruct()) { + // null literals for weird stuff like UDT's need + // special handling during type flattening, so + // don't use ValuesRel for those + return null; + } + + RexNode literalExpr = + exprConverter.convertLiteral( + bb, + (SqlLiteral) sqlNode); + + if (!(literalExpr instanceof RexLiteral)) { + assert literalExpr.isA(SqlKind.CAST); + RexNode child = ((RexCall) literalExpr).getOperands().get(0); + assert RexLiteral.isNullLiteral(child); + + // NOTE jvs 22-Nov-2006: we preserve type info + // in ValuesRel digest, so it's OK to lose it here + return (RexLiteral) child; + } + + RexLiteral literal = (RexLiteral) literalExpr; + + Comparable value = literal.getValue(); + + if (SqlTypeUtil.isExactNumeric(type)) { + BigDecimal roundedValue = + NumberUtil.rescaleBigDecimal( + (BigDecimal) value, + type.getScale()); + return rexBuilder.makeExactLiteral( + roundedValue, + type); + } + + if ((value instanceof NlsString) + && (type.getSqlTypeName() == SqlTypeName.CHAR)) { + // pad fixed character type + NlsString unpadded = (NlsString) value; + return rexBuilder.makeCharLiteral( + new NlsString( + Util.rpad(unpadded.getValue(), type.getPrecision()), + unpadded.getCharsetName(), + unpadded.getCollation())); + } + return literal; + } + + private boolean isRowConstructor(SqlNode node) { + if (!(node.getKind() == SqlKind.ROW)) { + return false; + } + SqlCall call = (SqlCall) node; + return call.getOperator().getName().equalsIgnoreCase("row"); + } + + /** + * Builds a list of all IN or EXISTS operators + * inside SQL parse tree. Does not traverse inside queries. + * + * @param bb blackboard + * @param node the SQL parse tree + * @param logic Whether the answer needs to be in full 3-valued logic (TRUE, + * FALSE, UNKNOWN) will be required, or whether we can accept + * an approximation (say representing UNKNOWN as FALSE) + * @param registerOnlyScalarSubqueries if set to true and the parse tree + * corresponds to a variation of a select + * node, only register it if it's a scalar + * subquery + */ +private void findSubqueries( + Blackboard bb, + SqlNode node, + RelOptUtil.Logic logic, + boolean registerOnlyScalarSubqueries) { + final SqlKind kind = node.getKind(); + switch (kind) { + case EXISTS: + case SELECT: + case MULTISET_QUERY_CONSTRUCTOR: + case MULTISET_VALUE_CONSTRUCTOR: + case CURSOR: + case SCALAR_QUERY: + if (!registerOnlyScalarSubqueries + || (kind == SqlKind.SCALAR_QUERY)) { + bb.registerSubquery(node, RelOptUtil.Logic.TRUE_FALSE); + } + return; + case IN: + if (((SqlCall) node).getOperator() == SqlStdOperatorTable.NOT_IN) { + logic = logic.negate(); + } + break; + case NOT: + logic = logic.negate(); + break; + } + if (node instanceof SqlCall) { + if (kind == SqlKind.OR + || kind == SqlKind.NOT) { + // It's always correct to outer join subquery with + // containing query; however, when predicates involve Or + // or NOT, outer join might be necessary. + bb.subqueryNeedsOuterJoin = true; + } + for (SqlNode operand : ((SqlCall) node).getOperandList()) { + if (operand != null) { + // In the case of an IN expression, locate scalar + // subqueries so we can convert them to constants + findSubqueries( + bb, + operand, + logic, + kind == SqlKind.IN || registerOnlyScalarSubqueries); + } + } + } else if (node instanceof SqlNodeList) { + for (SqlNode child : (SqlNodeList) node) { + findSubqueries( + bb, + child, + logic, + kind == SqlKind.IN || registerOnlyScalarSubqueries); + } + } + + // Now that we've located any scalar subqueries inside the IN + // expression, register the IN expression itself. We need to + // register the scalar subqueries first so they can be converted + // before the IN expression is converted. + if (kind == SqlKind.IN) { + if (logic == RelOptUtil.Logic.TRUE_FALSE_UNKNOWN + && !validator.getValidatedNodeType(node).isNullable()) { + logic = RelOptUtil.Logic.UNKNOWN_AS_FALSE; + } + // TODO: This conversion is only valid in the WHERE clause + if (logic == RelOptUtil.Logic.UNKNOWN_AS_FALSE + && !bb.subqueryNeedsOuterJoin) { + logic = RelOptUtil.Logic.TRUE; + } + bb.registerSubquery(node, logic); + } + } + + /** + * Converts an expression from {@link SqlNode} to {@link RexNode} format. + * + * @param node Expression to translate + * @return Converted expression + */ + public RexNode convertExpression( + SqlNode node) { + Map nameToTypeMap = Collections.emptyMap(); + Blackboard bb = + createBlackboard( + new ParameterScope((SqlValidatorImpl) validator, nameToTypeMap), + null); + return bb.convertExpression(node); + } + + /** + * Converts an expression from {@link SqlNode} to {@link RexNode} format, + * mapping identifier references to predefined expressions. + * + * @param node Expression to translate + * @param nameToNodeMap map from String to {@link RexNode}; when an + * {@link SqlIdentifier} is encountered, it is used as a + * key and translated to the corresponding value from + * this map + * @return Converted expression + */ + public RexNode convertExpression( + SqlNode node, + Map nameToNodeMap) { + final Map nameToTypeMap = + new HashMap(); + for (Map.Entry entry : nameToNodeMap.entrySet()) { + nameToTypeMap.put(entry.getKey(), entry.getValue().getType()); + } + Blackboard bb = + createBlackboard( + new ParameterScope((SqlValidatorImpl) validator, nameToTypeMap), + nameToNodeMap); + return bb.convertExpression(node); + } + + /** + * Converts a non-standard expression. + * + *

This method is an extension-point that derived classes can override. If + * this method returns a null result, the normal expression translation + * process will proceed. The default implementation always returns null. + * + * @param node Expression + * @param bb Blackboard + * @return null to proceed with the usual expression translation process + */ + protected RexNode convertExtendedExpression( + SqlNode node, + Blackboard bb) { + return null; + } + + private RexNode convertOver(Blackboard bb, SqlNode node) { + SqlCall call = (SqlCall) node; + SqlCall aggCall = call.operand(0); + SqlNode windowOrRef = call.operand(1); + final SqlWindow window = + validator.resolveWindow(windowOrRef, bb.scope, true); + final SqlNodeList partitionList = window.getPartitionList(); + final ImmutableList.Builder partitionKeys = + ImmutableList.builder(); + for (SqlNode partition : partitionList) { + partitionKeys.add(bb.convertExpression(partition)); + } + RexNode lowerBound = bb.convertExpression(window.getLowerBound()); + RexNode upperBound = bb.convertExpression(window.getUpperBound()); + SqlNodeList orderList = window.getOrderList(); + if ((orderList.size() == 0) && !window.isRows()) { + // A logical range requires an ORDER BY clause. Use the implicit + // ordering of this relation. There must be one, otherwise it would + // have failed validation. + orderList = bb.scope.getOrderList(); + if (orderList == null) { + throw new AssertionError( + "Relation should have sort key for implicit ORDER BY"); + } + } + final ImmutableList.Builder orderKeys = + ImmutableList.builder(); + final Set flags = EnumSet.noneOf(SqlKind.class); + for (SqlNode order : orderList) { + flags.clear(); + RexNode e = bb.convertSortExpression(order, flags); + orderKeys.add(new RexFieldCollation(e, flags)); + } + try { + Util.permAssert(bb.window == null, "already in window agg mode"); + bb.window = window; + RexNode rexAgg = exprConverter.convertCall(bb, aggCall); + rexAgg = + rexBuilder.ensureType( + validator.getValidatedNodeType(call), rexAgg, false); + + // Walk over the tree and apply 'over' to all agg functions. This is + // necessary because the returned expression is not necessarily a call + // to an agg function. For example, AVG(x) becomes SUM(x) / COUNT(x). + final RexShuttle visitor = + new HistogramShuttle( + partitionKeys.build(), orderKeys.build(), + RexWindowBound.create(window.getLowerBound(), lowerBound), + RexWindowBound.create(window.getUpperBound(), upperBound), + window); + return rexAgg.accept(visitor); + } finally { + bb.window = null; + } + } + + /** + * Converts a FROM clause into a relational expression. + * + * @param bb Scope within which to resolve identifiers + * @param from FROM clause of a query. Examples include: + * + *

    + *
  • a single table ("SALES.EMP"), + *
  • an aliased table ("EMP AS E"), + *
  • a list of tables ("EMP, DEPT"), + *
  • an ANSI Join expression ("EMP JOIN DEPT ON EMP.DEPTNO = + * DEPT.DEPTNO"), + *
  • a VALUES clause ("VALUES ('Fred', 20)"), + *
  • a query ("(SELECT * FROM EMP WHERE GENDER = 'F')"), + *
  • or any combination of the above. + *
+ */ + protected void convertFrom( + Blackboard bb, + SqlNode from) { + SqlCall call; + final SqlNode[] operands; + switch (from.getKind()) { + case AS: + operands = ((SqlBasicCall) from).getOperands(); + convertFrom(bb, operands[0]); + return; + + case WITH_ITEM: + convertFrom(bb, ((SqlWithItem) from).query); + return; + + case WITH: + convertFrom(bb, ((SqlWith) from).body); + return; + + case TABLESAMPLE: + operands = ((SqlBasicCall) from).getOperands(); + SqlSampleSpec sampleSpec = SqlLiteral.sampleValue(operands[1]); + if (sampleSpec instanceof SqlSampleSpec.SqlSubstitutionSampleSpec) { + String sampleName = + ((SqlSampleSpec.SqlSubstitutionSampleSpec) sampleSpec) + .getName(); + datasetStack.push(sampleName); + convertFrom(bb, operands[0]); + datasetStack.pop(); + } else if (sampleSpec instanceof SqlSampleSpec.SqlTableSampleSpec) { + SqlSampleSpec.SqlTableSampleSpec tableSampleSpec = + (SqlSampleSpec.SqlTableSampleSpec) sampleSpec; + convertFrom(bb, operands[0]); + RelOptSamplingParameters params = + new RelOptSamplingParameters( + tableSampleSpec.isBernoulli(), + tableSampleSpec.getSamplePercentage(), + tableSampleSpec.isRepeatable(), + tableSampleSpec.getRepeatableSeed()); + bb.setRoot(new SamplingRel(cluster, bb.root, params), false); + } else { + throw Util.newInternal( + "unknown TABLESAMPLE type: " + sampleSpec); + } + return; + + case IDENTIFIER: + final SqlValidatorNamespace fromNamespace = + validator.getNamespace(from).resolve(); + if (fromNamespace.getNode() != null) { + convertFrom(bb, fromNamespace.getNode()); + return; + } + final String datasetName = + datasetStack.isEmpty() ? null : datasetStack.peek(); + boolean[] usedDataset = {false}; + RelOptTable table = + SqlValidatorUtil.getRelOptTable( + fromNamespace, + catalogReader, + datasetName, + usedDataset); + final RelNode tableRel; + if (shouldConvertTableAccess) { + tableRel = toRel(table); + } else { + tableRel = new TableAccessRel(cluster, table); + } + bb.setRoot(tableRel, true); + if (usedDataset[0]) { + bb.setDataset(datasetName); + } + return; + + case JOIN: + final SqlJoin join = (SqlJoin) from; + final Blackboard fromBlackboard = + createBlackboard(validator.getJoinScope(from), null); + SqlNode left = join.getLeft(); + SqlNode right = join.getRight(); + final boolean isNatural = join.isNatural(); + final JoinType joinType = join.getJoinType(); + final Blackboard leftBlackboard = + createBlackboard( + Util.first(validator.getJoinScope(left), + ((DelegatingScope) bb.scope).getParent()), null); + final Blackboard rightBlackboard = + createBlackboard( + Util.first(validator.getJoinScope(right), + ((DelegatingScope) bb.scope).getParent()), null); + convertFrom(leftBlackboard, left); + RelNode leftRel = leftBlackboard.root; + convertFrom(rightBlackboard, right); + RelNode rightRel = rightBlackboard.root; + JoinRelType convertedJoinType = convertJoinType(joinType); + RexNode conditionExp; + if (isNatural) { + final List columnList = + SqlValidatorUtil.deriveNaturalJoinColumnList( + validator.getNamespace(left).getRowType(), + validator.getNamespace(right).getRowType()); + conditionExp = convertUsing(leftRel, rightRel, columnList); + } else { + conditionExp = + convertJoinCondition( + fromBlackboard, + join.getCondition(), + join.getConditionType(), + leftRel, + rightRel); + } + + final RelNode joinRel = + createJoin( + fromBlackboard, + leftRel, + rightRel, + conditionExp, + convertedJoinType); + bb.setRoot(joinRel, false); + return; + + case SELECT: + case INTERSECT: + case EXCEPT: + case UNION: + final RelNode rel = convertQueryRecursive(from, false, null); + bb.setRoot(rel, true); + return; + + case VALUES: + convertValuesImpl(bb, (SqlCall) from, null); + return; + + case UNNEST: + final SqlNode node = ((SqlCall) from).operand(0); + replaceSubqueries(bb, node, RelOptUtil.Logic.TRUE_FALSE_UNKNOWN); + final RelNode childRel = + RelOptUtil.createProject( + (null != bb.root) ? bb.root : new OneRowRel(cluster), + Collections.singletonList(bb.convertExpression(node)), + Collections.singletonList(validator.deriveAlias(node, 0)), + true); + + UncollectRel uncollectRel = + new UncollectRel(cluster, cluster.traitSetOf(Convention.NONE), + childRel); + bb.setRoot(uncollectRel, true); + return; + + case COLLECTION_TABLE: + call = (SqlCall) from; + + // Dig out real call; TABLE() wrapper is just syntactic. + assert call.getOperandList().size() == 1; + call = call.operand(0); + convertCollectionTable(bb, call); + return; + + default: + throw Util.newInternal("not a join operator " + from); + } + } + + protected void convertCollectionTable( + Blackboard bb, + SqlCall call) { + final SqlOperator operator = call.getOperator(); + if (operator == SqlStdOperatorTable.TABLESAMPLE) { + final String sampleName = + SqlLiteral.stringValue(call.operand(0)); + datasetStack.push(sampleName); + SqlCall cursorCall = call.operand(1); + SqlNode query = cursorCall.operand(0); + RelNode converted = convertQuery(query, false, false); + bb.setRoot(converted, false); + datasetStack.pop(); + return; + } + replaceSubqueries(bb, call, RelOptUtil.Logic.TRUE_FALSE_UNKNOWN); + + // Expand table macro if possible. It's more efficient than + // TableFunctionRel. + if (operator instanceof SqlUserDefinedTableMacro) { + final SqlUserDefinedTableMacro udf = + (SqlUserDefinedTableMacro) operator; + final TranslatableTable table = udf.getTable(typeFactory, + call.getOperandList()); + final RelDataType rowType = table.getRowType(typeFactory); + RelOptTable relOptTable = RelOptTableImpl.create(null, rowType, table); + RelNode converted = toRel(relOptTable); + bb.setRoot(converted, true); + return; + } + + Type elementType; + if (operator instanceof SqlUserDefinedTableFunction) { + SqlUserDefinedTableFunction udtf = (SqlUserDefinedTableFunction) operator; + elementType = udtf.getElementType(typeFactory, call.getOperandList()); + } else { + elementType = null; + } + + RexNode rexCall = bb.convertExpression(call); + final List inputs = bb.retrieveCursors(); + Set columnMappings = + getColumnMappings(operator); + TableFunctionRel callRel = + new TableFunctionRel( + cluster, + inputs, + rexCall, + elementType, + validator.getValidatedNodeType(call), + columnMappings); + bb.setRoot(callRel, true); + afterTableFunction(bb, call, callRel); + } + + protected void afterTableFunction( + SqlToRelConverter.Blackboard bb, + SqlCall call, + TableFunctionRel callRel) { + } + + private Set getColumnMappings(SqlOperator op) { + SqlReturnTypeInference rti = op.getReturnTypeInference(); + if (rti == null) { + return null; + } + if (rti instanceof TableFunctionReturnTypeInference) { + TableFunctionReturnTypeInference tfrti = + (TableFunctionReturnTypeInference) rti; + return tfrti.getColumnMappings(); + } else { + return null; + } + } + + protected RelNode createJoin( + Blackboard bb, + RelNode leftRel, + RelNode rightRel, + RexNode joinCond, + JoinRelType joinType) { + assert joinCond != null; + + Set correlatedVariables = RelOptUtil.getVariablesUsed(rightRel); + if (correlatedVariables.size() > 0) { + final List correlations = Lists.newArrayList(); + + for (String correlName : correlatedVariables) { + DeferredLookup lookup = mapCorrelToDeferred.get(correlName); + RexFieldAccess fieldAccess = lookup.getFieldAccess(correlName); + String originalRelName = lookup.getOriginalRelName(); + String originalFieldName = fieldAccess.getField().getName(); + + int[] nsIndexes = {-1}; + final SqlValidatorScope[] ancestorScopes = {null}; + SqlValidatorNamespace foundNs = + lookup.bb.scope.resolve( + originalRelName, + ancestorScopes, + nsIndexes); + + assert foundNs != null; + assert nsIndexes.length == 1; + + int childNamespaceIndex = nsIndexes[0]; + + SqlValidatorScope ancestorScope = ancestorScopes[0]; + boolean correlInCurrentScope = ancestorScope == bb.scope; + + if (correlInCurrentScope) { + int namespaceOffset = 0; + if (childNamespaceIndex > 0) { + // If not the first child, need to figure out the width + // of output types from all the preceding namespaces + assert ancestorScope instanceof ListScope; + List children = + ((ListScope) ancestorScope).getChildren(); + + for (int i = 0; i < childNamespaceIndex; i++) { + SqlValidatorNamespace child = children.get(i); + namespaceOffset += + child.getRowType().getFieldCount(); + } + } + + RelDataTypeField field = + catalogReader.field(foundNs.getRowType(), originalFieldName); + int pos = namespaceOffset + field.getIndex(); + + assert field.getType() + == lookup.getFieldAccess(correlName).getField().getType(); + + assert pos != -1; + + if (bb.mapRootRelToFieldProjection.containsKey(bb.root)) { + // bb.root is an aggregate and only projects group by + // keys. + Map exprProjection = + bb.mapRootRelToFieldProjection.get(bb.root); + + // subquery can reference group by keys projected from + // the root of the outer relation. + if (exprProjection.containsKey(pos)) { + pos = exprProjection.get(pos); + } else { + // correl not grouped + throw Util.newInternal( + "Identifier '" + originalRelName + "." + + originalFieldName + "' is not a group expr"); + } + } + + Correlation newCorVar = + new Correlation( + getCorrelOrdinal(correlName), + pos); + + correlations.add(newCorVar); + } + } + + if (!correlations.isEmpty()) { + return new CorrelatorRel( + rightRel.getCluster(), + leftRel, + rightRel, + joinCond, + correlations, + joinType); + } + } + + final List extraLeftExprs = new ArrayList(); + final List extraRightExprs = new ArrayList(); + final int leftCount = leftRel.getRowType().getFieldCount(); + final int rightCount = rightRel.getRowType().getFieldCount(); + if (!containsGet(joinCond)) { + joinCond = pushDownJoinConditions( + joinCond, leftCount, rightCount, extraLeftExprs, extraRightExprs); + } + if (!extraLeftExprs.isEmpty()) { + final List fields = + leftRel.getRowType().getFieldList(); + leftRel = RelOptUtil.createProject( + leftRel, + new AbstractList>() { + @Override + public int size() { + return leftCount + extraLeftExprs.size(); + } + + @Override + public Pair get(int index) { + if (index < leftCount) { + RelDataTypeField field = fields.get(index); + return Pair.of( + new RexInputRef(index, field.getType()), + field.getName()); + } else { + return Pair.of( + extraLeftExprs.get(index - leftCount), null); + } + } + }, + true); + } + if (!extraRightExprs.isEmpty()) { + final List fields = + rightRel.getRowType().getFieldList(); + final int newLeftCount = leftCount + extraLeftExprs.size(); + rightRel = RelOptUtil.createProject( + rightRel, + new AbstractList>() { + @Override + public int size() { + return rightCount + extraRightExprs.size(); + } + + @Override + public Pair get(int index) { + if (index < rightCount) { + RelDataTypeField field = fields.get(index); + return Pair.of( + new RexInputRef(index, field.getType()), + field.getName()); + } else { + return Pair.of( + RexUtil.shift( + extraRightExprs.get(index - rightCount), + -newLeftCount), + null); + } + } + }, + true); + } + RelNode join = createJoin( + leftRel, + rightRel, + joinCond, + joinType, + ImmutableSet.of()); + if (!extraLeftExprs.isEmpty() || !extraRightExprs.isEmpty()) { + Mappings.TargetMapping mapping = + Mappings.createShiftMapping( + leftCount + extraLeftExprs.size() + + rightCount + extraRightExprs.size(), + 0, 0, leftCount, + leftCount, leftCount + extraLeftExprs.size(), rightCount); + return RelOptUtil.project(join, mapping); + } + return join; + } + + private static boolean containsGet(RexNode node) { + try { + node.accept( + new RexVisitorImpl(true) { + @Override public Void visitCall(RexCall call) { + if (call.getOperator() == RexBuilder.GET_OPERATOR) { + throw Util.FoundOne.NULL; + } + return super.visitCall(call); + } + }); + return false; + } catch (Util.FoundOne e) { + return true; + } + } + + /** + * Pushes down parts of a join condition. For example, given + * "emp JOIN dept ON emp.deptno + 1 = dept.deptno", adds a project above + * "emp" that computes the expression + * "emp.deptno + 1". The resulting join condition is a simple combination + * of AND, equals, and input fields. + */ + private RexNode pushDownJoinConditions( + RexNode node, + int leftCount, + int rightCount, + List extraLeftExprs, + List extraRightExprs) { + switch (node.getKind()) { + case AND: + case OR: + case EQUALS: + RexCall call = (RexCall) node; + List list = new ArrayList(); + List operands = Lists.newArrayList(call.getOperands()); + for (int i = 0; i < operands.size(); i++) { + RexNode operand = operands.get(i); + final int left2 = leftCount + extraLeftExprs.size(); + final int right2 = rightCount + extraRightExprs.size(); + final RexNode e = + pushDownJoinConditions( + operand, + leftCount, + rightCount, + extraLeftExprs, + extraRightExprs); + final List remainingOperands = Util.skip(operands, i + 1); + final int left3 = leftCount + extraLeftExprs.size(); + final int right3 = rightCount + extraRightExprs.size(); + fix(remainingOperands, left2, left3); + fix(list, left2, left3); + list.add(e); + } + if (!list.equals(call.getOperands())) { + return call.clone(call.getType(), list); + } + return call; + case INPUT_REF: + case LITERAL: + return node; + default: + BitSet bits = RelOptUtil.InputFinder.bits(node); + final int mid = leftCount + extraLeftExprs.size(); + switch (Side.of(bits, mid)) { + case LEFT: + fix(extraRightExprs, mid, mid + 1); + extraLeftExprs.add(node); + return new RexInputRef(mid, node.getType()); + case RIGHT: + final int index2 = mid + rightCount + extraRightExprs.size(); + extraRightExprs.add(node); + return new RexInputRef(index2, node.getType()); + case BOTH: + case EMPTY: + default: + return node; + } + } + } + + private void fix(List operands, int before, int after) { + if (before == after) { + return; + } + for (int i = 0; i < operands.size(); i++) { + RexNode node = operands.get(i); + operands.set(i, RexUtil.shift(node, before, after - before)); + } + } + + /** + * Categorizes whether a bit set contains bits left and right of a + * line. + */ + enum Side { + LEFT, RIGHT, BOTH, EMPTY; + + static Side of(BitSet bitSet, int middle) { + final int firstBit = bitSet.nextSetBit(0); + if (firstBit < 0) { + return EMPTY; + } + if (firstBit >= middle) { + return RIGHT; + } + if (bitSet.nextSetBit(middle) < 0) { + return LEFT; + } + return BOTH; + } + } + + /** + * Determines whether a subquery is non-correlated. Note that a + * non-correlated subquery can contain correlated references, provided those + * references do not reference select statements that are parents of the + * subquery. + * + * @param subq the subquery + * @param bb blackboard used while converting the subquery, i.e., the + * blackboard of the parent query of this subquery + * @return true if the subquery is non-correlated. + */ + private boolean isSubqNonCorrelated(RelNode subq, Blackboard bb) { + Set correlatedVariables = RelOptUtil.getVariablesUsed(subq); + for (String correlName : correlatedVariables) { + DeferredLookup lookup = mapCorrelToDeferred.get(correlName); + String originalRelName = lookup.getOriginalRelName(); + + int[] nsIndexes = {-1}; + final SqlValidatorScope[] ancestorScopes = {null}; + SqlValidatorNamespace foundNs = + lookup.bb.scope.resolve( + originalRelName, + ancestorScopes, + nsIndexes); + + assert foundNs != null; + assert nsIndexes.length == 1; + + SqlValidatorScope ancestorScope = ancestorScopes[0]; + + // If the correlated reference is in a scope that's "above" the + // subquery, then this is a correlated subquery. + SqlValidatorScope parentScope = bb.scope; + do { + if (ancestorScope == parentScope) { + return false; + } + if (parentScope instanceof DelegatingScope) { + parentScope = ((DelegatingScope) parentScope).getParent(); + } else { + break; + } + } while (parentScope != null); + } + return true; + } + + /** + * Returns a list of fields to be prefixed to each relational expression. + * + * @return List of system fields + */ + protected List getSystemFields() { + return Collections.emptyList(); + } + + private RexNode convertJoinCondition( + Blackboard bb, + SqlNode condition, + JoinConditionType conditionType, + RelNode leftRel, + RelNode rightRel) { + if (condition == null) { + return rexBuilder.makeLiteral(true); + } + bb.setRoot(ImmutableList.of(leftRel, rightRel)); + replaceSubqueries(bb, condition, RelOptUtil.Logic.UNKNOWN_AS_FALSE); + switch (conditionType) { + case ON: + bb.setRoot(ImmutableList.of(leftRel, rightRel)); + return bb.convertExpression(condition); + case USING: + SqlNodeList list = (SqlNodeList) condition; + List nameList = new ArrayList(); + for (SqlNode columnName : list) { + final SqlIdentifier id = (SqlIdentifier) columnName; + String name = id.getSimple(); + nameList.add(name); + } + return convertUsing(leftRel, rightRel, nameList); + default: + throw Util.unexpected(conditionType); + } + } + + /** + * Returns an expression for matching columns of a USING clause or inferred + * from NATURAL JOIN. "a JOIN b USING (x, y)" becomes "a.x = b.x AND a.y = + * b.y". Returns null if the column list is empty. + * + * @param leftRel Left input to the join + * @param rightRel Right input to the join + * @param nameList List of column names to join on + * @return Expression to match columns from name list, or true if name list + * is empty + */ + private RexNode convertUsing( + RelNode leftRel, + RelNode rightRel, + List nameList) { + final List list = Lists.newArrayList(); + for (String name : nameList) { + final RelDataType leftRowType = leftRel.getRowType(); + RelDataTypeField leftField = catalogReader.field(leftRowType, name); + RexNode left = + rexBuilder.makeInputRef( + leftField.getType(), + leftField.getIndex()); + final RelDataType rightRowType = rightRel.getRowType(); + RelDataTypeField rightField = + catalogReader.field(rightRowType, name); + RexNode right = + rexBuilder.makeInputRef( + rightField.getType(), + leftRowType.getFieldList().size() + rightField.getIndex()); + RexNode equalsCall = + rexBuilder.makeCall( + SqlStdOperatorTable.EQUALS, + left, + right); + list.add(equalsCall); + } + return RexUtil.composeConjunction(rexBuilder, list, false); + } + + private static JoinRelType convertJoinType(JoinType joinType) { + switch (joinType) { + case COMMA: + case INNER: + case CROSS: + return JoinRelType.INNER; + case FULL: + return JoinRelType.FULL; + case LEFT: + return JoinRelType.LEFT; + case RIGHT: + return JoinRelType.RIGHT; + default: + throw Util.unexpected(joinType); + } + } + + /** + * Converts the SELECT, GROUP BY and HAVING clauses of an aggregate query. + * + *

This method extracts SELECT, GROUP BY and HAVING clauses, and creates + * an {@link AggConverter}, then delegates to {@link #createAggImpl}. + * Derived class may override this method to change any of those clauses or + * specify a different {@link AggConverter}. + * + * @param bb Scope within which to resolve identifiers + * @param select Query + * @param orderExprList Additional expressions needed to implement ORDER BY + */ + protected void convertAgg( + Blackboard bb, + SqlSelect select, + List orderExprList) { + assert bb.root != null : "precondition: child != null"; + SqlNodeList groupList = select.getGroup(); + SqlNodeList selectList = select.getSelectList(); + SqlNode having = select.getHaving(); + + final AggConverter aggConverter = new AggConverter(bb, select); + createAggImpl( + bb, + aggConverter, + selectList, + groupList, + having, + orderExprList); + } + + protected final void createAggImpl( + Blackboard bb, + AggConverter aggConverter, + SqlNodeList selectList, + SqlNodeList groupList, + SqlNode having, + List orderExprList) { + SqlNodeList aggList = new SqlNodeList(SqlParserPos.ZERO); + + for (SqlNode selectNode : selectList) { + if (validator.isAggregate(selectNode)) { + aggList.add(selectNode); + } + } + + // first replace the subqueries inside the aggregates + // because they will provide input rows to the aggregates. + replaceSubqueries(bb, aggList, RelOptUtil.Logic.TRUE_FALSE_UNKNOWN); + + // If group-by clause is missing, pretend that it has zero elements. + if (groupList == null) { + groupList = SqlNodeList.EMPTY; + } + + // register the group exprs + + // build a map to remember the projections from the top scope to the + // output of the current root. + // + // Currently farrago allows expressions, not just column references in + // group by list. This is not SQL 2003 compliant. + + Map groupExprProjection = + new HashMap(); + + int i = -1; + for (SqlNode groupExpr : groupList) { + ++i; + final SqlNode expandedGroupExpr = + validator.expand(groupExpr, bb.scope); + aggConverter.addGroupExpr(expandedGroupExpr); + + if (expandedGroupExpr instanceof SqlIdentifier) { + // SQL 2003 does not allow expressions of column references + SqlIdentifier expr = (SqlIdentifier) expandedGroupExpr; + + // column references should be fully qualified. + assert expr.names.size() == 2; + String originalRelName = expr.names.get(0); + String originalFieldName = expr.names.get(1); + + int[] nsIndexes = {-1}; + final SqlValidatorScope[] ancestorScopes = {null}; + SqlValidatorNamespace foundNs = + bb.scope.resolve( + originalRelName, + ancestorScopes, + nsIndexes); + + assert foundNs != null; + assert nsIndexes.length == 1; + int childNamespaceIndex = nsIndexes[0]; + + int namespaceOffset = 0; + + if (childNamespaceIndex > 0) { + // If not the first child, need to figure out the width of + // output types from all the preceding namespaces + assert ancestorScopes[0] instanceof ListScope; + List children = + ((ListScope) ancestorScopes[0]).getChildren(); + + for (int j = 0; j < childNamespaceIndex; j++) { + namespaceOffset += + children.get(j).getRowType().getFieldCount(); + } + } + + RelDataTypeField field = + catalogReader.field(foundNs.getRowType(), originalFieldName); + int origPos = namespaceOffset + field.getIndex(); + + groupExprProjection.put(origPos, i); + } + } + + RexNode havingExpr = null; + List selectExprs = new ArrayList(); + List selectNames = new ArrayList(); + + try { + Util.permAssert(bb.agg == null, "already in agg mode"); + bb.agg = aggConverter; + + // convert the select and having expressions, so that the + // agg converter knows which aggregations are required + + selectList.accept(aggConverter); + for (SqlNode expr : orderExprList) { + expr.accept(aggConverter); + } + if (having != null) { + having.accept(aggConverter); + } + + // compute inputs to the aggregator + List preExprs = aggConverter.getPreExprs(); + List preNames = aggConverter.getPreNames(); + + if (preExprs.size() == 0) { + // Special case for COUNT(*), where we can end up with no inputs + // at all. The rest of the system doesn't like 0-tuples, so we + // select a dummy constant here. + preExprs = + Collections.singletonList( + (RexNode) rexBuilder.makeExactLiteral(BigDecimal.ZERO)); + preNames = Collections.singletonList(null); + } + + final RelNode inputRel = bb.root; + + // Project the expressions required by agg and having. + bb.setRoot( + RelOptUtil.createProject( + inputRel, + preExprs, + preNames, + true), + false); + bb.mapRootRelToFieldProjection.put(bb.root, groupExprProjection); + + // REVIEW jvs 31-Oct-2007: doesn't the declaration of + // monotonicity here assume sort-based aggregation at + // the physical level? + + // Tell bb which of group columns are sorted. + bb.columnMonotonicities.clear(); + for (SqlNode groupItem : groupList) { + bb.columnMonotonicities.add( + bb.scope.getMonotonicity(groupItem)); + } + + // Add the aggregator + bb.setRoot( + createAggregate( + bb, + BitSets.range(aggConverter.groupExprs.size()), + aggConverter.getAggCalls()), + false); + + bb.mapRootRelToFieldProjection.put(bb.root, groupExprProjection); + + // Replace subqueries in having here and modify having to use + // the replaced expressions + if (having != null) { + SqlNode newHaving = pushDownNotForIn(having); + replaceSubqueries(bb, newHaving, RelOptUtil.Logic.UNKNOWN_AS_FALSE); + havingExpr = bb.convertExpression(newHaving); + if (havingExpr.isAlwaysTrue()) { + havingExpr = null; + } + } + + // Now convert the other subqueries in the select list. + // This needs to be done separately from the subquery inside + // any aggregate in the select list, and after the aggregate rel + // is allocated. + replaceSubqueries(bb, selectList, RelOptUtil.Logic.TRUE_FALSE_UNKNOWN); + + // Now subqueries in the entire select list have been converted. + // Convert the select expressions to get the final list to be + // projected. + int k = 0; + + // For select expressions, use the field names previously assigned + // by the validator. If we derive afresh, we might generate names + // like "EXPR$2" that don't match the names generated by the + // validator. This is especially the case when there are system + // fields; system fields appear in the relnode's rowtype but do not + // (yet) appear in the validator type. + final SelectScope selectScope = + SqlValidatorUtil.getEnclosingSelectScope(bb.scope); + final SqlValidatorNamespace selectNamespace = + validator.getNamespace(selectScope.getNode()); + final List names = + selectNamespace.getRowType().getFieldNames(); + int sysFieldCount = selectList.size() - names.size(); + for (SqlNode expr : selectList) { + selectExprs.add(bb.convertExpression(expr)); + selectNames.add( + k < sysFieldCount + ? validator.deriveAlias(expr, k++) + : names.get(k++ - sysFieldCount)); + } + + for (SqlNode expr : orderExprList) { + selectExprs.add(bb.convertExpression(expr)); + selectNames.add(validator.deriveAlias(expr, k++)); + } + } finally { + bb.agg = null; + } + + // implement HAVING (we have already checked that it is non-trivial) + if (havingExpr != null) { + bb.setRoot(RelOptUtil.createFilter(bb.root, havingExpr), false); + } + + // implement the SELECT list + bb.setRoot( + RelOptUtil.createProject( + bb.root, + selectExprs, + selectNames, + true), + false); + + // Tell bb which of group columns are sorted. + bb.columnMonotonicities.clear(); + for (SqlNode selectItem : selectList) { + bb.columnMonotonicities.add( + bb.scope.getMonotonicity(selectItem)); + } + } + + /** + * Creates an AggregateRel. + * + *

In case the aggregate rel changes the order in which it projects + * fields, the groupExprProjection parameter is provided, and + * the implementation of this method may modify it. + * + *

The sortedCount parameter is the number of expressions + * known to be monotonic. These expressions must be on the leading edge of + * the grouping keys. The default implementation of this method ignores this + * parameter. + * + * @param bb Blackboard + * @param groupSet Bit set of ordinals of grouping columns + * @param aggCalls Array of calls to aggregate functions + * @return AggregateRel + */ + protected RelNode createAggregate( + Blackboard bb, + BitSet groupSet, + List aggCalls) { + return new AggregateRel( + cluster, + bb.root, + groupSet, + aggCalls); + } + + public RexDynamicParam convertDynamicParam( + final SqlDynamicParam dynamicParam) { + // REVIEW jvs 8-Jan-2005: dynamic params may be encountered out of + // order. Should probably cross-check with the count from the parser + // at the end and make sure they all got filled in. Why doesn't List + // have a resize() method?!? Make this a utility. + while (dynamicParam.getIndex() >= dynamicParamSqlNodes.size()) { + dynamicParamSqlNodes.add(null); + } + + dynamicParamSqlNodes.set( + dynamicParam.getIndex(), + dynamicParam); + return rexBuilder.makeDynamicParam( + getDynamicParamType(dynamicParam.getIndex()), + dynamicParam.getIndex()); + } + + /** + * Creates a list of collations required to implement the ORDER BY clause, + * if there is one. Populates extraOrderExprs with any sort + * expressions which are not in the select clause. + * + * @param bb Scope within which to resolve identifiers + * @param select Select clause. Never null, because we invent a + * dummy SELECT if ORDER BY is applied to a set + * operation (UNION etc.) + * @param orderList Order by clause, may be null + * @param extraOrderExprs Sort expressions which are not in the select + * clause (output) + * @param collationList List of collations (output) + */ + protected void gatherOrderExprs( + Blackboard bb, + SqlSelect select, + SqlNodeList orderList, + List extraOrderExprs, + List collationList) { + // TODO: add validation rules to SqlValidator also + assert bb.root != null : "precondition: child != null"; + assert select != null; + if (orderList == null) { + return; + } + for (SqlNode orderItem : orderList) { + collationList.add( + convertOrderItem( + select, + orderItem, + extraOrderExprs, + RelFieldCollation.Direction.ASCENDING, + RelFieldCollation.NullDirection.UNSPECIFIED)); + } + } + + protected RelFieldCollation convertOrderItem( + SqlSelect select, + SqlNode orderItem, List extraExprs, + RelFieldCollation.Direction direction, + RelFieldCollation.NullDirection nullDirection) { + assert select != null; + // Handle DESC keyword, e.g. 'select a, b from t order by a desc'. + switch (orderItem.getKind()) { + case DESCENDING: + return convertOrderItem( + select, + ((SqlCall) orderItem).operand(0), + extraExprs, + RelFieldCollation.Direction.DESCENDING, + nullDirection); + case NULLS_FIRST: + return convertOrderItem( + select, + ((SqlCall) orderItem).operand(0), + extraExprs, + direction, + RelFieldCollation.NullDirection.FIRST); + case NULLS_LAST: + return convertOrderItem( + select, + ((SqlCall) orderItem).operand(0), + extraExprs, + direction, + RelFieldCollation.NullDirection.LAST); + } + + SqlNode converted = validator.expandOrderExpr(select, orderItem); + + // Scan the select list and order exprs for an identical expression. + final SelectScope selectScope = validator.getRawSelectScope(select); + int ordinal = -1; + for (SqlNode selectItem : selectScope.getExpandedSelectList()) { + ++ordinal; + if (converted.equalsDeep(stripAs(selectItem), false)) { + return new RelFieldCollation( + ordinal, direction, nullDirection); + } + } + + for (SqlNode extraExpr : extraExprs) { + ++ordinal; + if (converted.equalsDeep(extraExpr, false)) { + return new RelFieldCollation( + ordinal, direction, nullDirection); + } + } + + // TODO: handle collation sequence + // TODO: flag expressions as non-standard + + extraExprs.add(converted); + return new RelFieldCollation(ordinal + 1, direction, nullDirection); + } + + protected boolean enableDecorrelation() { + // disable subquery decorrelation when needed. + // e.g. if outer joins are not supported. + return decorrelationEnabled; + } + + protected RelNode decorrelateQuery(RelNode rootRel) { + return RelDecorrelator.decorrelateQuery(rootRel); + } + + /** + * Sets whether to trim unused fields as part of the conversion process. + * + * @param trim Whether to trim unused fields + */ + public void setTrimUnusedFields(boolean trim) { + this.trimUnusedFields = trim; + } + + /** + * Returns whether to trim unused fields as part of the conversion process. + * + * @return Whether to trim unused fields + */ + public boolean isTrimUnusedFields() { + // OVERRIDE POINT + return false; // was `trimUnusedFields` + } + + /** + * Recursively converts a query to a relational expression. + * + * @param query Query + * @param top Whether this query is the top-level query of the + * statement + * @param targetRowType Target row type, or null + * @return Relational expression + */ + protected RelNode convertQueryRecursive( + SqlNode query, + boolean top, + RelDataType targetRowType) { + switch (query.getKind()) { + case SELECT: + return convertSelect((SqlSelect) query); + case INSERT: + return convertInsert((SqlInsert) query); + case DELETE: + return convertDelete((SqlDelete) query); + case UPDATE: + return convertUpdate((SqlUpdate) query); + case MERGE: + return convertMerge((SqlMerge) query); + case UNION: + case INTERSECT: + case EXCEPT: + return convertSetOp((SqlCall) query); + case WITH: + return convertWith((SqlWith) query); + case VALUES: + return convertValues((SqlCall) query, targetRowType); + default: + throw Util.newInternal("not a query: " + query); + } + } + + /** + * Converts a set operation (UNION, INTERSECT, MINUS) into relational + * expressions. + * + * @param call Call to set operator + * @return Relational expression + */ + protected RelNode convertSetOp(SqlCall call) { + final RelNode left = convertQueryRecursive(call.operand(0), false, null); + final RelNode right = convertQueryRecursive(call.operand(1), false, null); + boolean all = false; + if (call.getOperator() instanceof SqlSetOperator) { + all = ((SqlSetOperator) (call.getOperator())).isAll(); + } + switch (call.getKind()) { + case UNION: + return new UnionRel( + cluster, + ImmutableList.of(left, right), + all); + + case INTERSECT: + // TODO: all + if (!all) { + return new IntersectRel( + cluster, + ImmutableList.of(left, right), + all); + } else { + throw Util.newInternal( + "set operator INTERSECT ALL not suported"); + } + + case EXCEPT: + // TODO: all + if (!all) { + return new MinusRel( + cluster, + ImmutableList.of(left, right), + all); + } else { + throw Util.newInternal( + "set operator EXCEPT ALL not suported"); + } + + default: + throw Util.unexpected(call.getKind()); + } + } + + protected RelNode convertInsert(SqlInsert call) { + RelOptTable targetTable = getTargetTable(call); + + final RelDataType targetRowType = + validator.getValidatedNodeType(call); + assert targetRowType != null; + RelNode sourceRel = + convertQueryRecursive( + call.getSource(), + false, + targetRowType); + RelNode massagedRel = convertColumnList(call, sourceRel); + + final ModifiableTable modifiableTable = + targetTable.unwrap(ModifiableTable.class); + if (modifiableTable != null) { + return modifiableTable.toModificationRel( + cluster, + targetTable, + catalogReader, + massagedRel, + TableModificationRel.Operation.INSERT, + null, + false); + } + return new TableModificationRel( + cluster, + targetTable, + catalogReader, + massagedRel, + TableModificationRel.Operation.INSERT, + null, + false); + } + + private RelOptTable.ToRelContext createToRelContext() { + return new RelOptTable.ToRelContext() { + public RelOptCluster getCluster() { + return cluster; + } + + public RelNode expandView( + RelDataType rowType, + String queryString, + List schemaPath) { + return viewExpander.expandView(rowType, queryString, schemaPath); + } + }; + } + + public RelNode toRel(RelOptTable table) { + return table.toRel(createToRelContext()); + } + + protected RelOptTable getTargetTable(SqlNode call) { + SqlValidatorNamespace targetNs = validator.getNamespace(call).resolve(); + return SqlValidatorUtil.getRelOptTable(targetNs, catalogReader, null, null); + } + + /** + * Creates a source for an INSERT statement. + * + *

If the column list is not specified, source expressions match target + * columns in order. + * + *

If the column list is specified, Source expressions are mapped to + * target columns by name via targetColumnList, and may not cover the entire + * target table. So, we'll make up a full row, using a combination of + * default values and the source expressions provided. + * + * @param call Insert expression + * @param sourceRel Source relational expression + * @return Converted INSERT statement + */ + protected RelNode convertColumnList( + SqlInsert call, + RelNode sourceRel) { + RelDataType sourceRowType = sourceRel.getRowType(); + final RexNode sourceRef = + rexBuilder.makeRangeReference(sourceRowType, 0, false); + final List targetColumnNames = new ArrayList(); + final List columnExprs = new ArrayList(); + collectInsertTargets(call, sourceRef, targetColumnNames, columnExprs); + + final RelOptTable targetTable = getTargetTable(call); + final RelDataType targetRowType = targetTable.getRowType(); + final List targetFields = + targetRowType.getFieldList(); + final List sourceExps = + new ArrayList( + Collections.nCopies(targetFields.size(), null)); + final List fieldNames = + new ArrayList( + Collections.nCopies(targetFields.size(), null)); + + // Walk the name list and place the associated value in the + // expression list according to the ordinal value returned from + // the table construct, leaving nulls in the list for columns + // that are not referenced. + for (Pair p : Pair.zip(targetColumnNames, columnExprs)) { + RelDataTypeField field = catalogReader.field(targetRowType, p.left); + assert field != null : "column " + p.left + " not found"; + sourceExps.set(field.getIndex(), p.right); + } + + // Walk the expression list and get default values for any columns + // that were not supplied in the statement. Get field names too. + for (int i = 0; i < targetFields.size(); ++i) { + final RelDataTypeField field = targetFields.get(i); + final String fieldName = field.getName(); + fieldNames.set(i, fieldName); + if (sourceExps.get(i) != null) { + if (defaultValueFactory.isGeneratedAlways(targetTable, i)) { + throw RESOURCE.insertIntoAlwaysGenerated(fieldName).ex(); + } + continue; + } + sourceExps.set( + i, defaultValueFactory.newColumnDefaultValue(targetTable, i)); + + // bare nulls are dangerous in the wrong hands + sourceExps.set( + i, + castNullLiteralIfNeeded( + sourceExps.get(i), field.getType())); + } + + return RelOptUtil.createProject(sourceRel, sourceExps, fieldNames, true); + } + + private RexNode castNullLiteralIfNeeded(RexNode node, RelDataType type) { + if (!RexLiteral.isNullLiteral(node)) { + return node; + } + return rexBuilder.makeCast(type, node); + } + + /** + * Given an INSERT statement, collects the list of names to be populated and + * the expressions to put in them. + * + * @param call Insert statement + * @param sourceRef Expression representing a row from the source + * relational expression + * @param targetColumnNames List of target column names, to be populated + * @param columnExprs List of expressions, to be populated + */ + protected void collectInsertTargets( + SqlInsert call, + final RexNode sourceRef, + final List targetColumnNames, + List columnExprs) { + final RelOptTable targetTable = getTargetTable(call); + final RelDataType targetRowType = targetTable.getRowType(); + SqlNodeList targetColumnList = call.getTargetColumnList(); + if (targetColumnList == null) { + targetColumnNames.addAll(targetRowType.getFieldNames()); + } else { + for (int i = 0; i < targetColumnList.size(); i++) { + SqlIdentifier id = (SqlIdentifier) targetColumnList.get(i); + targetColumnNames.add(id.getSimple()); + } + } + + for (int i = 0; i < targetColumnNames.size(); i++) { + final RexNode expr = rexBuilder.makeFieldAccess(sourceRef, i); + columnExprs.add(expr); + } + } + + private RelNode convertDelete(SqlDelete call) { + RelOptTable targetTable = getTargetTable(call); + RelNode sourceRel = convertSelect(call.getSourceSelect()); + return new TableModificationRel( + cluster, + targetTable, + catalogReader, + sourceRel, + TableModificationRel.Operation.DELETE, + null, + false); + } + + private RelNode convertUpdate(SqlUpdate call) { + RelOptTable targetTable = getTargetTable(call); + + // convert update column list from SqlIdentifier to String + List targetColumnNameList = new ArrayList(); + for (SqlNode node : call.getTargetColumnList()) { + SqlIdentifier id = (SqlIdentifier) node; + String name = id.getSimple(); + targetColumnNameList.add(name); + } + + RelNode sourceRel = convertSelect(call.getSourceSelect()); + + return new TableModificationRel( + cluster, + targetTable, + catalogReader, + sourceRel, + TableModificationRel.Operation.UPDATE, + targetColumnNameList, + false); + } + + private RelNode convertMerge(SqlMerge call) { + RelOptTable targetTable = getTargetTable(call); + + // convert update column list from SqlIdentifier to String + List targetColumnNameList = new ArrayList(); + SqlUpdate updateCall = call.getUpdateCall(); + if (updateCall != null) { + for (SqlNode targetColumn : updateCall.getTargetColumnList()) { + SqlIdentifier id = (SqlIdentifier) targetColumn; + String name = id.getSimple(); + targetColumnNameList.add(name); + } + } + + // replace the projection of the source select with a + // projection that contains the following: + // 1) the expressions corresponding to the new insert row (if there is + // an insert) + // 2) all columns from the target table (if there is an update) + // 3) the set expressions in the update call (if there is an update) + + // first, convert the merge's source select to construct the columns + // from the target table and the set expressions in the update call + RelNode mergeSourceRel = convertSelect(call.getSourceSelect()); + + // then, convert the insert statement so we can get the insert + // values expressions + SqlInsert insertCall = call.getInsertCall(); + int nLevel1Exprs = 0; + List level1InsertExprs = null; + List level2InsertExprs = null; + if (insertCall != null) { + RelNode insertRel = convertInsert(insertCall); + + // if there are 2 level of projections in the insert source, combine + // them into a single project; level1 refers to the topmost project; + // the level1 projection contains references to the level2 + // expressions, except in the case where no target expression was + // provided, in which case, the expression is the default value for + // the column; or if the expressions directly map to the source + // table + level1InsertExprs = + ((ProjectRel) insertRel.getInput(0)).getProjects(); + if (insertRel.getInput(0).getInput(0) instanceof ProjectRel) { + level2InsertExprs = + ((ProjectRel) insertRel.getInput(0).getInput(0)) + .getProjects(); + } + nLevel1Exprs = level1InsertExprs.size(); + } + + JoinRel joinRel = (JoinRel) mergeSourceRel.getInput(0); + int nSourceFields = joinRel.getLeft().getRowType().getFieldCount(); + List projects = new ArrayList(); + for (int level1Idx = 0; level1Idx < nLevel1Exprs; level1Idx++) { + if ((level2InsertExprs != null) + && (level1InsertExprs.get(level1Idx) instanceof RexInputRef)) { + int level2Idx = + ((RexInputRef) level1InsertExprs.get(level1Idx)).getIndex(); + projects.add(level2InsertExprs.get(level2Idx)); + } else { + projects.add(level1InsertExprs.get(level1Idx)); + } + } + if (updateCall != null) { + final ProjectRel project = (ProjectRel) mergeSourceRel; + projects.addAll( + Util.skip(project.getProjects(), nSourceFields)); + } + + RelNode massagedRel = + RelOptUtil.createProject(joinRel, projects, null, true); + + return new TableModificationRel( + cluster, + targetTable, + catalogReader, + massagedRel, + TableModificationRel.Operation.MERGE, + targetColumnNameList, + false); + } + + /** + * Converts an identifier into an expression in a given scope. For example, + * the "empno" in "select empno from emp join dept" becomes "emp.empno". + */ + private RexNode convertIdentifier( + Blackboard bb, + SqlIdentifier identifier) { + // first check for reserved identifiers like CURRENT_USER + final SqlCall call = SqlUtil.makeCall(opTab, identifier); + if (call != null) { + return bb.convertExpression(call); + } + + if (bb.agg != null) { + throw Util.newInternal("Identifier '" + identifier + + "' is not a group expr"); + } + + SqlValidatorNamespace namespace = null; + if (bb.scope != null) { + identifier = bb.scope.fullyQualify(identifier); + namespace = bb.scope.resolve(identifier.names.get(0), null, null); + } + RexNode e = bb.lookupExp(identifier.names.get(0)); + final String correlationName; + if (e instanceof RexCorrelVariable) { + correlationName = ((RexCorrelVariable) e).getName(); + } else { + correlationName = null; + } + + for (String name : Util.skip(identifier.names)) { + if (namespace != null) { + name = namespace.translate(name); + namespace = null; + } + final boolean caseSensitive = true; // name already fully-qualified + e = rexBuilder.makeFieldAccess(e, name, caseSensitive); + } + if (e instanceof RexInputRef) { + // adjust the type to account for nulls introduced by outer joins + e = adjustInputRef(bb, (RexInputRef) e); + } + + if (null != correlationName) { + // REVIEW: make mapCorrelateVariableToRexNode map to RexFieldAccess + assert e instanceof RexFieldAccess; + final RexNode prev = + bb.mapCorrelateVariableToRexNode.put(correlationName, e); + assert prev == null; + } + return e; + } + + /** + * Adjusts the type of a reference to an input field to account for nulls + * introduced by outer joins; and adjusts the offset to match the physical + * implementation. + * + * @param bb Blackboard + * @param inputRef Input ref + * @return Adjusted input ref + */ + protected RexNode adjustInputRef( + Blackboard bb, + RexInputRef inputRef) { + RelDataTypeField field = bb.getRootField(inputRef); + if (field != null) { + return rexBuilder.makeInputRef( + field.getType(), + inputRef.getIndex()); + } + return inputRef; + } + + /** + * Converts a row constructor into a relational expression. + * + * @param bb Blackboard + * @param rowConstructor Row constructor expression + * @return Relational expression which returns a single row. + * @pre isRowConstructor(rowConstructor) + */ + private RelNode convertRowConstructor( + Blackboard bb, + SqlCall rowConstructor) { + assert isRowConstructor(rowConstructor) : rowConstructor; + final List operands = rowConstructor.getOperandList(); + return convertMultisets(operands, bb); + } + + private RelNode convertCursor(Blackboard bb, SubQuery subQuery) { + final SqlCall cursorCall = (SqlCall) subQuery.node; + assert cursorCall.operandCount() == 1; + SqlNode query = cursorCall.operand(0); + RelNode converted = convertQuery(query, false, false); + int iCursor = bb.cursors.size(); + bb.cursors.add(converted); + subQuery.expr = + new RexInputRef( + iCursor, + converted.getRowType()); + return converted; + } + + private RelNode convertMultisets(final List operands, + Blackboard bb) { + // NOTE: Wael 2/04/05: this implementation is not the most efficient in + // terms of planning since it generates XOs that can be reduced. + List joinList = new ArrayList(); + List lastList = new ArrayList(); + for (int i = 0; i < operands.size(); i++) { + SqlNode operand = operands.get(i); + if (!(operand instanceof SqlCall)) { + lastList.add(operand); + continue; + } + + final SqlCall call = (SqlCall) operand; + final SqlOperator op = call.getOperator(); + if ((op != SqlStdOperatorTable.MULTISET_VALUE) + && (op != SqlStdOperatorTable.MULTISET_QUERY)) { + lastList.add(operand); + continue; + } + final RelNode input; + if (op == SqlStdOperatorTable.MULTISET_VALUE) { + final SqlNodeList list = + new SqlNodeList(call.getOperandList(), call.getParserPosition()); +// assert bb.scope instanceof SelectScope : bb.scope; + CollectNamespace nss = + (CollectNamespace) validator.getNamespace(call); + Blackboard usedBb; + if (null != nss) { + usedBb = createBlackboard(nss.getScope(), null); + } else { + usedBb = + createBlackboard( + new ListScope(bb.scope) { + public SqlNode getNode() { + return call; + } + }, + null); + } + RelDataType multisetType = validator.getValidatedNodeType(call); + validator.setValidatedNodeType( + list, + multisetType.getComponentType()); + input = convertQueryOrInList(usedBb, list); + } else { + input = convertQuery(call.operand(0), false, true); + } + + if (lastList.size() > 0) { + joinList.add(lastList); + } + lastList = new ArrayList(); + CollectRel collectRel = + new CollectRel( + cluster, + cluster.traitSetOf(Convention.NONE), + input, + validator.deriveAlias(call, i)); + joinList.add(collectRel); + } + + if (joinList.size() == 0) { + joinList.add(lastList); + } + + for (int i = 0; i < joinList.size(); i++) { + Object o = joinList.get(i); + if (o instanceof List) { + List projectList = (List) o; + final List selectList = new ArrayList(); + final List fieldNameList = new ArrayList(); + for (int j = 0; j < projectList.size(); j++) { + SqlNode operand = projectList.get(j); + selectList.add(bb.convertExpression(operand)); + + // REVIEW angel 5-June-2005: Use deriveAliasFromOrdinal + // instead of deriveAlias to match field names from + // SqlRowOperator. Otherwise, get error Type + // 'RecordType(INTEGER EMPNO)' has no field 'EXPR$0' when + // doing select * from unnest( select multiset[empno] + // from sales.emps); + + fieldNameList.add(SqlUtil.deriveAliasFromOrdinal(j)); + } + + RelNode projRel = + RelOptUtil.createProject( + new OneRowRel(cluster), + selectList, + fieldNameList); + + joinList.set(i, projRel); + } + } + + RelNode ret = (RelNode) joinList.get(0); + for (int i = 1; i < joinList.size(); i++) { + RelNode relNode = (RelNode) joinList.get(i); + ret = + createJoin( + ret, + relNode, + rexBuilder.makeLiteral(true), + JoinRelType.INNER, + ImmutableSet.of()); + } + return ret; + } + + /** + * Factory method that creates a join. + * A subclass can override to use a different kind of join. + * + * @param left Left input + * @param right Right input + * @param condition Join condition + * @param joinType Join type + * @param variablesStopped Set of names of variables which are set by the + * LHS and used by the RHS and are not available to + * nodes above this JoinRel in the tree + * @return A relational expression representing a join + */ + protected RelNode createJoin( + RelNode left, + RelNode right, + RexNode condition, + JoinRelType joinType, + Set variablesStopped) { + return new JoinRel( + cluster, + left, + right, + condition, + joinType, + variablesStopped); + } + + private void convertSelectList( + Blackboard bb, + SqlSelect select, + List orderList) { + SqlNodeList selectList = select.getSelectList(); + selectList = validator.expandStar(selectList, select, false); + + replaceSubqueries(bb, selectList, RelOptUtil.Logic.TRUE_FALSE_UNKNOWN); + + List fieldNames = new ArrayList(); + List exprs = new ArrayList(); + Collection aliases = new TreeSet(); + + // Project any system fields. (Must be done before regular select items, + // because offsets may be affected.) + final List columnMonotonicityList = + new ArrayList(); + extraSelectItems( + bb, + select, + exprs, + fieldNames, + aliases, + columnMonotonicityList); + + // Project select clause. + int i = -1; + for (SqlNode expr : selectList) { + ++i; + exprs.add(bb.convertExpression(expr)); + fieldNames.add(deriveAlias(expr, aliases, i)); + } + + // Project extra fields for sorting. + for (SqlNode expr : orderList) { + ++i; + SqlNode expr2 = validator.expandOrderExpr(select, expr); + exprs.add(bb.convertExpression(expr2)); + fieldNames.add(deriveAlias(expr, aliases, i)); + } + + fieldNames = SqlValidatorUtil.uniquify(fieldNames); + + RelNode inputRel = bb.root; + bb.setRoot( + RelOptUtil.createProject(bb.root, exprs, fieldNames), + false); + + assert bb.columnMonotonicities.isEmpty(); + bb.columnMonotonicities.addAll(columnMonotonicityList); + for (SqlNode selectItem : selectList) { + bb.columnMonotonicities.add( + selectItem.getMonotonicity(bb.scope)); + } + } + + /** + * Adds extra select items. The default implementation adds nothing; derived + * classes may add columns to exprList, nameList, aliasList and + * columnMonotonicityList. + * + * @param bb Blackboard + * @param select Select statement being translated + * @param exprList List of expressions in select clause + * @param nameList List of names, one per column + * @param aliasList Collection of aliases that have been used + * already + * @param columnMonotonicityList List of monotonicity, one per column + */ + protected void extraSelectItems( + Blackboard bb, + SqlSelect select, + List exprList, + List nameList, + Collection aliasList, + List columnMonotonicityList) { + } + + private String deriveAlias( + final SqlNode node, + Collection aliases, + final int ordinal) { + String alias = validator.deriveAlias(node, ordinal); + if ((alias == null) || aliases.contains(alias)) { + String aliasBase = (alias == null) ? "EXPR$" : alias; + for (int j = 0;; j++) { + alias = aliasBase + j; + if (!aliases.contains(alias)) { + break; + } + } + } + aliases.add(alias); + return alias; + } + + /** + * Converts a WITH sub-query into a relational expression. + */ + public RelNode convertWith(SqlWith with) { + return convertQuery(with.body, false, false); + } + + /** + * Converts a SELECT statement's parse tree into a relational expression. + */ + public RelNode convertValues( + SqlCall values, + RelDataType targetRowType) { + final SqlValidatorScope scope = validator.getOverScope(values); + assert scope != null; + final Blackboard bb = createBlackboard(scope, null); + convertValuesImpl(bb, values, targetRowType); + return bb.root; + } + + /** + * Converts a values clause (as in "INSERT INTO T(x,y) VALUES (1,2)") into a + * relational expression. + * + * @param bb Blackboard + * @param values Call to SQL VALUES operator + * @param targetRowType Target row type + */ + private void convertValuesImpl( + Blackboard bb, + SqlCall values, + RelDataType targetRowType) { + // Attempt direct conversion to ValuesRel; if that fails, deal with + // fancy stuff like subqueries below. + RelNode valuesRel = + convertRowValues( + bb, + values, + values.getOperandList(), + true, + targetRowType); + if (valuesRel != null) { + bb.setRoot(valuesRel, true); + return; + } + + List unionRels = new ArrayList(); + for (SqlNode rowConstructor1 : values.getOperandList()) { + SqlCall rowConstructor = (SqlCall) rowConstructor1; + Blackboard tmpBb = createBlackboard(bb.scope, null); + replaceSubqueries(tmpBb, rowConstructor, + RelOptUtil.Logic.TRUE_FALSE_UNKNOWN); + List> exps = + new ArrayList>(); + for (Ord operand : Ord.zip(rowConstructor.getOperandList())) { + exps.add( + Pair.of( + tmpBb.convertExpression(operand.e), + validator.deriveAlias(operand.e, operand.i))); + } + RelNode in = + (null == tmpBb.root) + ? new OneRowRel(cluster) + : tmpBb.root; + unionRels.add( + RelOptUtil.createProject( + in, + Pair.left(exps), + Pair.right(exps), + true)); + } + + if (unionRels.size() == 0) { + throw Util.newInternal("empty values clause"); + } else if (unionRels.size() == 1) { + bb.setRoot( + unionRels.get(0), + true); + } else { + bb.setRoot( + new UnionRel( + cluster, + unionRels, + true), + true); + } + + // REVIEW jvs 22-Jan-2004: should I add + // mapScopeToLux.put(validator.getScope(values),bb.root); + // ? + } + + private String createCorrel() { + int n = nextCorrel++; + return CORREL_PREFIX + n; + } + + private int getCorrelOrdinal(String correlName) { + assert correlName.startsWith(CORREL_PREFIX); + return Integer.parseInt(correlName.substring(CORREL_PREFIX.length())); + } + + //~ Inner Classes ---------------------------------------------------------- + + /** + * Workspace for translating an individual SELECT statement (or sub-SELECT). + */ + protected class Blackboard implements SqlRexContext, SqlVisitor { + /** + * Collection of {@link RelNode} objects which correspond to a SELECT + * statement. + */ + public final SqlValidatorScope scope; + private final Map nameToNodeMap; + public RelNode root; + private List inputs; + private final Map mapCorrelateVariableToRexNode = + new HashMap(); + + List cursors; + + /** + * List of IN and EXISTS nodes inside this + * SELECT statement (but not inside sub-queries). + */ + private final Set subqueryList = Sets.newLinkedHashSet(); + + private final Map subqueryMap = + Util.asIndexMap(subqueryList, FN); + + private boolean subqueryNeedsOuterJoin; + + /** + * Workspace for building aggregates. + */ + AggConverter agg; + + /** + * When converting window aggregate, we need to know if the window is + * guaranteed to be non-empty. + */ + SqlWindow window; + + /** + * Project the groupby expressions out of the root of this sub-select. + * Subqueries can reference group by expressions projected from the + * "right" to the subquery. + */ + private final Map> + mapRootRelToFieldProjection = + new HashMap>(); + + private final List columnMonotonicities = + new ArrayList(); + + private final List systemFieldList = + new ArrayList(); + + /** + * Creates a Blackboard. + * + * @param scope Name-resolution scope for expressions validated + * within this query. Can be null if this Blackboard is + * for a leaf node, say + * @param nameToNodeMap Map which translates the expression to map a + * given parameter into, if translating expressions; + * null otherwise + */ + protected Blackboard( + SqlValidatorScope scope, + Map nameToNodeMap) { + this.scope = scope; + this.nameToNodeMap = nameToNodeMap; + this.cursors = new ArrayList(); + subqueryNeedsOuterJoin = false; + } + + public RexNode register( + RelNode rel, + JoinRelType joinType) { + return register(rel, joinType, null); + } + + /** + * Registers a relational expression. + * + * @param rel Relational expression + * @param joinType Join type + * @param leftKeys LHS of IN clause, or null for expressions + * other than IN + * @return Expression with which to refer to the row (or partial row) + * coming from this relational expression's side of the join + */ + public RexNode register( + RelNode rel, + JoinRelType joinType, + List leftKeys) { + assert joinType != null; + if (root == null) { + assert leftKeys == null; + setRoot(rel, false); + return rexBuilder.makeRangeReference( + root.getRowType(), + 0, + false); + } + + final RexNode joinCond; + final int origLeftInputCount = root.getRowType().getFieldCount(); + if (leftKeys != null) { + List newLeftInputExpr = Lists.newArrayList(); + for (int i = 0; i < origLeftInputCount; i++) { + newLeftInputExpr.add(rexBuilder.makeInputRef(root, i)); + } + + final List leftJoinKeys = Lists.newArrayList(); + for (RexNode leftKey : leftKeys) { + newLeftInputExpr.add(leftKey); + leftJoinKeys.add(origLeftInputCount + leftJoinKeys.size()); + } + + ProjectRel newLeftInput = + (ProjectRel) RelOptUtil.createProject( + root, + newLeftInputExpr, + null, + true); + + // maintain the group by mapping in the new ProjectRel + if (mapRootRelToFieldProjection.containsKey(root)) { + mapRootRelToFieldProjection.put( + newLeftInput, + mapRootRelToFieldProjection.get(root)); + } + + setRoot(newLeftInput, false); + + // right fields appear after the LHS fields. + final int rightOffset = root.getRowType().getFieldCount() + - newLeftInput.getRowType().getFieldCount(); + final List rightKeys = + Util.range(rightOffset, rightOffset + leftJoinKeys.size()); + + joinCond = + RelOptUtil.createEquiJoinCondition(newLeftInput, leftJoinKeys, + rel, rightKeys, rexBuilder); + } else { + joinCond = rexBuilder.makeLiteral(true); + } + + int leftFieldCount = root.getRowType().getFieldCount(); + final RelNode join = + createJoin( + this, + root, + rel, + joinCond, + joinType); + + setRoot(join, false); + + if (leftKeys != null + && joinType == JoinRelType.LEFT) { + final int leftKeyCount = leftKeys.size(); + int rightFieldLength = rel.getRowType().getFieldCount(); + assert leftKeyCount == rightFieldLength - 1; + + final int rexRangeRefLength = leftKeyCount + rightFieldLength; + RelDataType returnType = + typeFactory.createStructType( + new AbstractList>() { + public Map.Entry get( + int index) { + return join.getRowType().getFieldList() + .get(origLeftInputCount + index); + } + + public int size() { + return rexRangeRefLength; + } + }); + + return rexBuilder.makeRangeReference( + returnType, + origLeftInputCount, + false); + } else { + return rexBuilder.makeRangeReference( + rel.getRowType(), + leftFieldCount, + joinType.generatesNullsOnRight()); + } + } + + /** + * Sets a new root relational expression, as the translation process + * backs its way further up the tree. + * + * @param root New root relational expression + * @param leaf Whether the relational expression is a leaf, that is, + * derived from an atomic relational expression such as a table + * name in the from clause, or the projection on top of a + * select-subquery. In particular, relational expressions + * derived from JOIN operators are not leaves, but set + * expressions are. + */ + public void setRoot(RelNode root, boolean leaf) { + setRoot( + Collections.singletonList(root), root, root instanceof JoinRel); + if (leaf) { + leaves.add(root); + } + this.columnMonotonicities.clear(); + } + + private void setRoot( + List inputs, + RelNode root, + boolean hasSystemFields) { + this.inputs = inputs; + this.root = root; + this.systemFieldList.clear(); + if (hasSystemFields) { + this.systemFieldList.addAll(getSystemFields()); + } + } + + /** + * Notifies this Blackboard that the root just set using {@link + * #setRoot(RelNode, boolean)} was derived using dataset substitution. + * + *

The default implementation is not interested in such + * notifications, and does nothing. + * + * @param datasetName Dataset name + */ + public void setDataset(String datasetName) { + } + + void setRoot(List inputs) { + setRoot(inputs, null, false); + } + + /** + * Returns an expression with which to reference a from-list item. + * + * @param name the alias of the from item + * @return a {@link RexFieldAccess} or {@link RexRangeRef}, or null if + * not found + */ + RexNode lookupExp(String name) { + if (nameToNodeMap != null) { + RexNode node = nameToNodeMap.get(name); + if (node == null) { + throw Util.newInternal( + "Unknown identifier '" + name + + "' encountered while expanding expression" + node); + } + return node; + } + int[] offsets = {-1}; + final SqlValidatorScope[] ancestorScopes = {null}; + SqlValidatorNamespace foundNs = + scope.resolve(name, ancestorScopes, offsets); + if (foundNs == null) { + return null; + } + + // Found in current query's from list. Find which from item. + // We assume that the order of the from clause items has been + // preserved. + SqlValidatorScope ancestorScope = ancestorScopes[0]; + boolean isParent = ancestorScope != scope; + if ((inputs != null) && !isParent) { + int offset = offsets[0]; + final LookupContext rels = + new LookupContext(this, inputs, systemFieldList.size()); + return lookup(offset, rels); + } else { + // We're referencing a relational expression which has not been + // converted yet. This occurs when from items are correlated, + // e.g. "select from emp as emp join emp.getDepts() as dept". + // Create a temporary expression. + assert isParent; + DeferredLookup lookup = new DeferredLookup(this, name); + String correlName = createCorrel(); + mapCorrelToDeferred.put(correlName, lookup); + final RelDataType rowType = foundNs.getRowType(); + return rexBuilder.makeCorrel(rowType, correlName); + } + } + + /** + * Creates an expression with which to reference the expression whose + * offset in its from-list is {@code offset}. + */ + RexNode lookup( + int offset, + LookupContext lookupContext) { + Pair pair = lookupContext.findRel(offset); + return rexBuilder.makeRangeReference( + pair.left.getRowType(), + pair.right, + false); + } + + RelDataTypeField getRootField(RexInputRef inputRef) { + int fieldOffset = inputRef.getIndex(); + for (RelNode input : inputs) { + RelDataType rowType = input.getRowType(); + if (rowType == null) { + // TODO: remove this once leastRestrictive + // is correctly implemented + return null; + } + if (fieldOffset < rowType.getFieldCount()) { + return rowType.getFieldList().get(fieldOffset); + } + fieldOffset -= rowType.getFieldCount(); + } + throw new AssertionError(); + } + + public void flatten( + List rels, + int systemFieldCount, + int[] start, + List> relOffsetList) { + for (RelNode rel : rels) { + if (leaves.contains(rel)) { + relOffsetList.add( + Pair.of(rel, start[0])); + start[0] += rel.getRowType().getFieldCount(); + } else { + if (rel instanceof JoinRel + || rel instanceof AggregateRel) { + start[0] += systemFieldCount; + } + flatten( + rel.getInputs(), + systemFieldCount, + start, + relOffsetList); + } + } + } + + void registerSubquery(SqlNode node, RelOptUtil.Logic logic) { + subqueryList.add(new SubQuery(node, logic)); + } + + ImmutableList retrieveCursors() { + try { + return ImmutableList.copyOf(cursors); + } finally { + cursors.clear(); + } + } + + // implement SqlRexContext + public RexNode convertExpression(SqlNode expr) { + // If we're in aggregation mode and this is an expression in the + // GROUP BY clause, return a reference to the field. + if (agg != null) { + final SqlNode expandedGroupExpr = validator.expand(expr, scope); + RexNode rex = agg.lookupGroupExpr(expandedGroupExpr); + if (rex != null) { + return rex; + } + if (expr instanceof SqlCall) { + rex = agg.lookupAggregates((SqlCall) expr); + if (rex != null) { + return rex; + } + } + } + + // Allow the derived class chance to override the standard + // behavior for special kinds of expressions. + RexNode rex = convertExtendedExpression(expr, this); + if (rex != null) { + return rex; + } + + boolean needTruthTest; + + // Sub-queries and OVER expressions are not like ordinary + // expressions. + final SqlKind kind = expr.getKind(); + final SubQuery subQuery; + switch (kind) { + case CURSOR: + case SELECT: + case EXISTS: + case SCALAR_QUERY: + subQuery = subqueryMap.get(expr); + assert subQuery != null; + rex = subQuery.expr; + assert rex != null : "rex != null"; + + if (kind == SqlKind.CURSOR) { + // cursor reference is pre-baked + return rex; + } + if (((kind == SqlKind.SCALAR_QUERY) + || (kind == SqlKind.EXISTS)) + && isConvertedSubq(rex)) { + // scalar subquery or EXISTS has been converted to a + // constant + return rex; + } + + RexNode fieldAccess; + needTruthTest = false; + + // The indicator column is the last field of the subquery. + fieldAccess = + rexBuilder.makeFieldAccess( + rex, + rex.getType().getFieldCount() - 1); + + // The indicator column will be nullable if it comes from + // the null-generating side of the join. For EXISTS, add an + // "IS TRUE" check so that the result is "BOOLEAN NOT NULL". + if (fieldAccess.getType().isNullable()) { + if (kind == SqlKind.EXISTS) { + needTruthTest = true; + } + } + + if (needTruthTest) { + fieldAccess = + rexBuilder.makeCall( + SqlStdOperatorTable.IS_NOT_NULL, + fieldAccess); + } + return fieldAccess; + + case IN: + subQuery = subqueryMap.get(expr); + assert subQuery != null; + assert subQuery.expr != null : "expr != null"; + return subQuery.expr; + + case OVER: + return convertOver(this, expr); + + default: + // fall through + } + + // Apply standard conversions. + rex = expr.accept(this); + Util.permAssert(rex != null, "conversion result not null"); + return rex; + } + + /** + * Converts an item in an ORDER BY clause, extracting DESC, NULLS LAST + * and NULLS FIRST flags first. + */ + public RexNode convertSortExpression(SqlNode expr, Set flags) { + switch (expr.getKind()) { + case DESCENDING: + case NULLS_LAST: + case NULLS_FIRST: + flags.add(expr.getKind()); + final SqlNode operand = ((SqlCall) expr).operand(0); + return convertSortExpression(operand, flags); + default: + return convertExpression(expr); + } + } + + /** + * Determines whether a RexNode corresponds to a subquery that's been + * converted to a constant. + * + * @param rex the expression to be examined + * @return true if the expression is a dynamic parameter, a literal, or + * a literal that is being cast + */ + private boolean isConvertedSubq(RexNode rex) { + if ((rex instanceof RexLiteral) + || (rex instanceof RexDynamicParam)) { + return true; + } + if (rex instanceof RexCall) { + RexCall call = (RexCall) rex; + if (call.getOperator() == SqlStdOperatorTable.CAST) { + RexNode operand = call.getOperands().get(0); + if (operand instanceof RexLiteral) { + return true; + } + } + } + return false; + } + + // implement SqlRexContext + public int getGroupCount() { + if (agg != null) { + return agg.groupExprs.size(); + } + if (window != null) { + return window.isAlwaysNonEmpty() ? 1 : 0; + } + return -1; + } + + // implement SqlRexContext + public RexBuilder getRexBuilder() { + return rexBuilder; + } + + // implement SqlRexContext + public RexRangeRef getSubqueryExpr(SqlCall call) { + final SubQuery subQuery = subqueryMap.get(call); + assert subQuery != null; + return (RexRangeRef) subQuery.expr; + } + + // implement SqlRexContext + public RelDataTypeFactory getTypeFactory() { + return typeFactory; + } + + // implement SqlRexContext + public DefaultValueFactory getDefaultValueFactory() { + return defaultValueFactory; + } + + // implement SqlRexContext + public SqlValidator getValidator() { + return validator; + } + + // implement SqlRexContext + public RexNode convertLiteral(SqlLiteral literal) { + return exprConverter.convertLiteral(this, literal); + } + + public RexNode convertInterval(SqlIntervalQualifier intervalQualifier) { + return exprConverter.convertInterval(this, intervalQualifier); + } + + // implement SqlVisitor + public RexNode visit(SqlLiteral literal) { + return exprConverter.convertLiteral(this, literal); + } + + // implement SqlVisitor + public RexNode visit(SqlCall call) { + if (agg != null) { + final SqlOperator op = call.getOperator(); + if (op.isAggregator()) { + return agg.lookupAggregates(call); + } + } + return exprConverter.convertCall(this, call); + } + + // implement SqlVisitor + public RexNode visit(SqlNodeList nodeList) { + throw new UnsupportedOperationException(); + } + + // implement SqlVisitor + public RexNode visit(SqlIdentifier id) { + return convertIdentifier(this, id); + } + + // implement SqlVisitor + public RexNode visit(SqlDataTypeSpec type) { + throw new UnsupportedOperationException(); + } + + // implement SqlVisitor + public RexNode visit(SqlDynamicParam param) { + return convertDynamicParam(param); + } + + // implement SqlVisitor + public RexNode visit(SqlIntervalQualifier intervalQualifier) { + return convertInterval(intervalQualifier); + } + + public List getColumnMonotonicities() { + return columnMonotonicities; + } + } + + private static class DeferredLookup { + Blackboard bb; + String originalRelName; + + DeferredLookup( + Blackboard bb, + String originalRelName) { + this.bb = bb; + this.originalRelName = originalRelName; + } + + public RexFieldAccess getFieldAccess(String name) { + return (RexFieldAccess) bb.mapCorrelateVariableToRexNode.get(name); + } + + public String getOriginalRelName() { + return originalRelName; + } + } + + /** + * An implementation of DefaultValueFactory which always supplies NULL. + */ + class NullDefaultValueFactory implements DefaultValueFactory { + public boolean isGeneratedAlways( + RelOptTable table, + int iColumn) { + return false; + } + + public RexNode newColumnDefaultValue( + RelOptTable table, + int iColumn) { + return rexBuilder.constantNull(); + } + + public RexNode newAttributeInitializer( + RelDataType type, + SqlFunction constructor, + int iAttribute, + List constructorArgs) { + return rexBuilder.constantNull(); + } + } + + /** + * A default implementation of SubqueryConverter that does no conversion. + */ + private class NoOpSubqueryConverter implements SubqueryConverter { + // implement SubqueryConverter + public boolean canConvertSubquery() { + return false; + } + + // implement SubqueryConverter + public RexNode convertSubquery( + SqlCall subquery, + SqlToRelConverter parentConverter, + boolean isExists, + boolean isExplain) { + throw new IllegalArgumentException(); + } + } + + /** + * Converts expressions to aggregates. + * + *

Consider the expression SELECT deptno, SUM(2 * sal) FROM emp GROUP BY + * deptno Then + * + *

    + *
  • groupExprs = {SqlIdentifier(deptno)}
    • + *
  • convertedInputExprs = {RexInputRef(deptno), 2 * + * RefInputRef(sal)}
    • + *
  • inputRefs = {RefInputRef(#0), RexInputRef(#1)}
    • + *
  • aggCalls = {AggCall(SUM, {1})}
    • + *
+ */ + protected class AggConverter implements SqlVisitor { + private final Blackboard bb; + + private final Map nameMap = + new HashMap(); + + /** + * The group-by expressions, in {@link SqlNode} format. + */ + private final SqlNodeList groupExprs = + new SqlNodeList(SqlParserPos.ZERO); + + /** + * Input expressions for the group columns and aggregates, in {@link + * RexNode} format. The first elements of the list correspond to the + * elements in {@link #groupExprs}; the remaining elements are for + * aggregates. + */ + private final List convertedInputExprs = + new ArrayList(); + + /** + * Names of {@link #convertedInputExprs}, where the expressions are + * simple mappings to input fields. + */ + private final List convertedInputExprNames = + new ArrayList(); + + private final List inputRefs = + new ArrayList(); + private final List aggCalls = + new ArrayList(); + private final Map aggMapping = + new HashMap(); + private final Map aggCallMapping = + new HashMap(); + + /** + * Creates an AggConverter. + * + *

The select parameter provides enough context to name + * aggregate calls which are top-level select list items. + * + * @param bb Blackboard + * @param select Query being translated; provides context to give + */ + public AggConverter(Blackboard bb, SqlSelect select) { + this.bb = bb; + + // Collect all expressions used in the select list so that aggregate + // calls can be named correctly. + final SqlNodeList selectList = select.getSelectList(); + for (int i = 0; i < selectList.size(); i++) { + SqlNode selectItem = selectList.get(i); + String name = null; + if (SqlUtil.isCallTo( + selectItem, + SqlStdOperatorTable.AS)) { + final SqlCall call = (SqlCall) selectItem; + selectItem = call.operand(0); + name = call.operand(1).toString(); + } + if (name == null) { + name = validator.deriveAlias(selectItem, i); + } + nameMap.put(selectItem.toString(), name); + } + } + + public void addGroupExpr(SqlNode expr) { + RexNode convExpr = bb.convertExpression(expr); + final RexNode rex = lookupGroupExpr(expr); + if (rex != null) { + return; // don't add duplicates, in e.g. "GROUP BY x, y, x" + } + groupExprs.add(expr); + String name = nameMap.get(expr.toString()); + addExpr(convExpr, name); + final RelDataType type = convExpr.getType(); + inputRefs.add(rexBuilder.makeInputRef(type, inputRefs.size())); + } + + /** + * Adds an expression, deducing an appropriate name if possible. + * + * @param expr Expression + * @param name Suggested name + */ + private void addExpr(RexNode expr, String name) { + convertedInputExprs.add(expr); + if ((name == null) && (expr instanceof RexInputRef)) { + final int i = ((RexInputRef) expr).getIndex(); + name = bb.root.getRowType().getFieldList().get(i).getName(); + } + if (convertedInputExprNames.contains(name)) { + // In case like 'SELECT ... GROUP BY x, y, x', don't add + // name 'x' twice. + name = null; + } + convertedInputExprNames.add(name); + } + + // implement SqlVisitor + public Void visit(SqlIdentifier id) { + return null; + } + + // implement SqlVisitor + public Void visit(SqlNodeList nodeList) { + for (int i = 0; i < nodeList.size(); i++) { + nodeList.get(i).accept(this); + } + return null; + } + + // implement SqlVisitor + public Void visit(SqlLiteral lit) { + return null; + } + + // implement SqlVisitor + public Void visit(SqlDataTypeSpec type) { + return null; + } + + // implement SqlVisitor + public Void visit(SqlDynamicParam param) { + return null; + } + + // implement SqlVisitor + public Void visit(SqlIntervalQualifier intervalQualifier) { + return null; + } + + public Void visit(SqlCall call) { + if (call.getOperator().isAggregator()) { + assert bb.agg == this; + List args = new ArrayList(); + List argTypes = + call.getOperator() instanceof SqlCountAggFunction + ? new ArrayList(call.getOperandList().size()) + : null; + try { + // switch out of agg mode + bb.agg = null; + for (SqlNode operand : call.getOperandList()) { + RexNode convertedExpr; + + // special case for COUNT(*): delete the * + if (operand instanceof SqlIdentifier) { + SqlIdentifier id = (SqlIdentifier) operand; + if (id.isStar()) { + assert call.operandCount() == 1; + assert args.isEmpty(); + break; + } + } + convertedExpr = bb.convertExpression(operand); + assert convertedExpr != null; + if (argTypes != null) { + argTypes.add(convertedExpr.getType()); + } + args.add(lookupOrCreateGroupExpr(convertedExpr)); + } + } finally { + // switch back into agg mode + bb.agg = this; + } + + final Aggregation aggregation = + (Aggregation) call.getOperator(); + RelDataType type = validator.deriveType(bb.scope, call); + boolean distinct = false; + SqlLiteral quantifier = call.getFunctionQuantifier(); + if ((null != quantifier) + && (quantifier.getValue() == SqlSelectKeyword.DISTINCT)) { + distinct = true; + } + final AggregateCall aggCall = + new AggregateCall( + aggregation, + distinct, + args, + type, + nameMap.get(call.toString())); + RexNode rex = + rexBuilder.addAggCall( + aggCall, + groupExprs.size(), + aggCalls, + aggCallMapping, + argTypes); + aggMapping.put(call, rex); + } else if (call instanceof SqlSelect) { + // rchen 2006-10-17: + // for now do not detect aggregates in subqueries. + return null; + } else { + for (SqlNode operand : call.getOperandList()) { + // Operands are occasionally null, e.g. switched CASE arg 0. + if (operand != null) { + operand.accept(this); + } + } + } + return null; + } + + private int lookupOrCreateGroupExpr(RexNode expr) { + for (int i = 0; i < convertedInputExprs.size(); i++) { + RexNode convertedInputExpr = convertedInputExprs.get(i); + if (expr.toString().equals(convertedInputExpr.toString())) { + return i; + } + } + + // not found -- add it + int index = convertedInputExprs.size(); + addExpr(expr, null); + return index; + } + + /** + * If an expression is structurally identical to one of the group-by + * expressions, returns a reference to the expression, otherwise returns + * null. + */ + public RexNode lookupGroupExpr(SqlNode expr) { + for (int i = 0; i < groupExprs.size(); i++) { + SqlNode groupExpr = groupExprs.get(i); + if (expr.equalsDeep(groupExpr, false)) { + return inputRefs.get(i); + } + } + return null; + } + + public RexNode lookupAggregates(SqlCall call) { + // assert call.getOperator().isAggregator(); + assert bb.agg == this; + + return aggMapping.get(call); + } + + public List getPreExprs() { + return convertedInputExprs; + } + + public List getPreNames() { + return convertedInputExprNames; + } + + public List getAggCalls() { + return aggCalls; + } + + public RelDataTypeFactory getTypeFactory() { + return typeFactory; + } + } + + /** + * Context to find a relational expression to a field offset. + */ + private static class LookupContext { + private final List> relOffsetList = + new ArrayList>(); + + /** + * Creates a LookupContext with multiple input relational expressions. + * + * @param bb Context for translating this subquery + * @param rels Relational expressions + * @param systemFieldCount Number of system fields + */ + LookupContext(Blackboard bb, List rels, int systemFieldCount) { + bb.flatten(rels, systemFieldCount, new int[]{0}, relOffsetList); + } + + /** + * Returns the relational expression with a given offset, and the + * ordinal in the combined row of its first field. + * + *

For example, in {@code Emp JOIN Dept}, findRel(1) returns the + * relational expression for {@code Dept} and offset 6 (because + * {@code Emp} has 6 fields, therefore the first field of {@code Dept} + * is field 6. + * + * @param offset Offset of relational expression in FROM clause + * @return Relational expression and the ordinal of its first field + */ + Pair findRel(int offset) { + return relOffsetList.get(offset); + } + } + + /** + * Shuttle which walks over a tree of {@link RexNode}s and applies 'over' to + * all agg functions. + * + *

This is necessary because the returned expression is not necessarily a + * call to an agg function. For example, + * + *

AVG(x)
+ * + * becomes + * + *
SUM(x) / COUNT(x)
+ * + *

Any aggregate functions are converted to calls to the internal + * $Histogram aggregation function and accessors such as + * $HistogramMin; for example, + * + *

MIN(x), MAX(x)
+ * + * are converted to + * + *
$HistogramMin($Histogram(x)), + * $HistogramMax($Histogram(x))
+ * + * Common sub-expression elmination will ensure that only one histogram is + * computed. + */ + private class HistogramShuttle extends RexShuttle { + /** + * Whether to convert calls to MIN(x) to HISTOGRAM_MIN(HISTOGRAM(x)). + * Histograms allow rolling computation, but require more space. + */ + static final boolean ENABLE_HISTOGRAM_AGG = false; + + private final List partitionKeys; + private final ImmutableList orderKeys; + private final RexWindowBound lowerBound; + private final RexWindowBound upperBound; + private final SqlWindow window; + + HistogramShuttle( + List partitionKeys, + ImmutableList orderKeys, + RexWindowBound lowerBound, RexWindowBound upperBound, + SqlWindow window) { + this.partitionKeys = partitionKeys; + this.orderKeys = orderKeys; + this.lowerBound = lowerBound; + this.upperBound = upperBound; + this.window = window; + } + + public RexNode visitCall(RexCall call) { + final SqlOperator op = call.getOperator(); + if (!(op instanceof SqlAggFunction)) { + return super.visitCall(call); + } + final SqlAggFunction aggOp = (SqlAggFunction) op; + final RelDataType type = call.getType(); + List exprs = call.getOperands(); + + SqlFunction histogramOp = !ENABLE_HISTOGRAM_AGG + ? null + : getHistogramOp(aggOp); + + if (histogramOp != null) { + final RelDataType histogramType = computeHistogramType(type); + + // For DECIMAL, since it's already represented as a bigint we + // want to do a reinterpretCast instead of a cast to avoid + // losing any precision. + boolean reinterpretCast = + type.getSqlTypeName() == SqlTypeName.DECIMAL; + + // Replace original expression with CAST of not one + // of the supported types + if (histogramType != type) { + exprs = new ArrayList(exprs); + exprs.set( + 0, + reinterpretCast + ? rexBuilder.makeReinterpretCast(histogramType, exprs.get(0), + rexBuilder.makeLiteral(false)) + : rexBuilder.makeCast(histogramType, exprs.get(0))); + } + + RexCallBinding bind = + new RexCallBinding( + rexBuilder.getTypeFactory(), + SqlStdOperatorTable.HISTOGRAM_AGG, + exprs); + + RexNode over = + rexBuilder.makeOver( + SqlStdOperatorTable.HISTOGRAM_AGG + .inferReturnType(bind), + SqlStdOperatorTable.HISTOGRAM_AGG, + exprs, + partitionKeys, + orderKeys, + lowerBound, + upperBound, + window.isRows(), + window.isAllowPartial(), + false); + + RexNode histogramCall = + rexBuilder.makeCall( + histogramType, + histogramOp, + ImmutableList.of(over)); + + // If needed, post Cast result back to original + // type. + if (histogramType != type) { + if (reinterpretCast) { + histogramCall = + rexBuilder.makeReinterpretCast( + type, + histogramCall, + rexBuilder.makeLiteral(false)); + } else { + histogramCall = + rexBuilder.makeCast(type, histogramCall); + } + } + + return histogramCall; + } else { + boolean needSum0 = aggOp == SqlStdOperatorTable.SUM + && type.isNullable(); + SqlAggFunction aggOpToUse = + needSum0 ? SqlStdOperatorTable.SUM0 + : aggOp; + return rexBuilder.makeOver( + type, + aggOpToUse, + exprs, + partitionKeys, + orderKeys, + lowerBound, + upperBound, + window.isRows(), + window.isAllowPartial(), + needSum0); + } + } + + /** + * Returns the histogram operator corresponding to a given aggregate + * function. + * + *

For example, getHistogramOp({@link + * SqlStdOperatorTable#MIN}} returns {@link + * SqlStdOperatorTable#HISTOGRAM_MIN}. + * + * @param aggFunction An aggregate function + * @return Its histogram function, or null + */ + SqlFunction getHistogramOp(SqlAggFunction aggFunction) { + if (aggFunction == SqlStdOperatorTable.MIN) { + return SqlStdOperatorTable.HISTOGRAM_MIN; + } else if (aggFunction == SqlStdOperatorTable.MAX) { + return SqlStdOperatorTable.HISTOGRAM_MAX; + } else if (aggFunction == SqlStdOperatorTable.FIRST_VALUE) { + return SqlStdOperatorTable.HISTOGRAM_FIRST_VALUE; + } else if (aggFunction == SqlStdOperatorTable.LAST_VALUE) { + return SqlStdOperatorTable.HISTOGRAM_LAST_VALUE; + } else { + return null; + } + } + + /** + * Returns the type for a histogram function. It is either the actual + * type or an an approximation to it. + */ + private RelDataType computeHistogramType(RelDataType type) { + if (SqlTypeUtil.isExactNumeric(type) + && type.getSqlTypeName() != SqlTypeName.BIGINT) { + return typeFactory.createSqlType(SqlTypeName.BIGINT); + } else if (SqlTypeUtil.isApproximateNumeric(type) + && type.getSqlTypeName() != SqlTypeName.DOUBLE) { + return typeFactory.createSqlType(SqlTypeName.DOUBLE); + } else { + return type; + } + } + } + + /** A sub-query, whether it needs to be translated using 2- or 3-valued + * logic. */ + private static class SubQuery { + final SqlNode node; + final RelOptUtil.Logic logic; + RexNode expr; + + private SubQuery(SqlNode node, RelOptUtil.Logic logic) { + this.node = node; + this.logic = logic; + } + } +} + +// End SqlToRelConverter.java diff --git a/pom.xml b/pom.xml index 28242f8..c38a8cc 100644 --- a/pom.xml +++ b/pom.xml @@ -497,6 +497,7 @@ + atopcalcite common metadata dictionary diff --git a/query/pom.xml b/query/pom.xml index 189eb82..0f035b3 100644 --- a/query/pom.xml +++ b/query/pom.xml @@ -19,16 +19,13 @@ com.kylinolap - kylin-storage + atopcalcite ${project.parent.version} - org.apache.calcite - calcite-core - - - org.apache.calcite - calcite-avatica + com.kylinolap + kylin-storage + ${project.parent.version} net.hydromatic diff --git a/query/src/main/java/net/hydromatic/optiq/runtime/SqlFunctions.java b/query/src/main/java/net/hydromatic/optiq/runtime/SqlFunctions.java deleted file mode 100644 index 6f209f5..0000000 --- a/query/src/main/java/net/hydromatic/optiq/runtime/SqlFunctions.java +++ /dev/null @@ -1,1705 +0,0 @@ -/* - * OVERRIDE POINTS: - * - divide(BigDecimal,BigDecimal), was `b0.divide(b1)`, now `b0.divide(b1, MathContext.DECIMAL64);` - */ - -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to you under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package net.hydromatic.optiq.runtime; - -import net.hydromatic.avatica.ByteString; - -import net.hydromatic.linq4j.Enumerable; -import net.hydromatic.linq4j.Linq4j; -import net.hydromatic.linq4j.expressions.Primitive; -import net.hydromatic.linq4j.function.Deterministic; -import net.hydromatic.linq4j.function.Function1; -import net.hydromatic.linq4j.function.NonDeterministic; - -import net.hydromatic.optiq.DataContext; - -import org.eigenbase.util14.DateTimeUtil; - -import java.math.*; -import java.sql.SQLException; -import java.sql.Timestamp; -import java.text.DecimalFormat; -import java.util.*; -import java.util.regex.Pattern; - -/** - * Helper methods to implement SQL functions in generated code. - * - *

Not present: and, or, not (builtin operators are better, because they - * use lazy evaluation. Implementations do not check for null values; the - * calling code must do that.

- * - *

Many of the functions do not check for null values. This is intentional. - * If null arguments are possible, the code-generation framework checks for - * nulls before calling the functions.

- */ -@SuppressWarnings({"unused", "rawtypes", "unchecked"}) -@Deterministic -public class SqlFunctions { - private static final DecimalFormat DOUBLE_FORMAT = - new DecimalFormat("0.0E0"); - - /** The julian date of the epoch, 1970-01-01. */ - public static final int EPOCH_JULIAN = 2440588; - - private static final TimeZone LOCAL_TZ = TimeZone.getDefault(); - - private static final Function1, Enumerable> - LIST_AS_ENUMERABLE = - new Function1, Enumerable>() { - public Enumerable apply(List list) { - return Linq4j.asEnumerable(list); - } - }; - - private SqlFunctions() { - } - - /** SQL SUBSTRING(string FROM ... FOR ...) function. */ - public static String substring(String s, int from, int for_) { - return s.substring(from - 1, Math.min(from - 1 + for_, s.length())); - } - - /** SQL SUBSTRING(string FROM ...) function. */ - public static String substring(String s, int from) { - return s.substring(from - 1); - } - - /** SQL UPPER(string) function. */ - public static String upper(String s) { - return s.toUpperCase(); - } - - /** SQL LOWER(string) function. */ - public static String lower(String s) { - return s.toLowerCase(); - } - - /** SQL INITCAP(string) function. */ - public static String initcap(String s) { - // Assumes Alpha as [A-Za-z0-9] - // white space is treated as everything else. - final int len = s.length(); - boolean start = true; - final StringBuilder newS = new StringBuilder(); - - for (int i = 0; i < len; i++) { - char curCh = s.charAt(i); - final int c = (int) curCh; - if (start) { // curCh is whitespace or first character of word. - if (c > 47 && c < 58) { // 0-9 - start = false; - } else if (c > 64 && c < 91) { // A-Z - start = false; - } else if (c > 96 && c < 123) { // a-z - start = false; - curCh = (char) (c - 32); // Uppercase this character - } - // else {} whitespace - } else { // Inside of a word or white space after end of word. - if (c > 47 && c < 58) { // 0-9 - // noop - } else if (c > 64 && c < 91) { // A-Z - curCh = (char) (c + 32); // Lowercase this character - } else if (c > 96 && c < 123) { // a-z - // noop - } else { // whitespace - start = true; - } - } - newS.append(curCh); - } // for each character in s - return newS.toString(); - } - - /** SQL CHARACTER_LENGTH(string) function. */ - public static int charLength(String s) { - return s.length(); - } - - /** SQL {@code string || string} operator. */ - public static String concat(String s0, String s1) { - return s0 + s1; - } - - /** SQL {@code binary || binary} operator. */ - public static ByteString concat(ByteString s0, ByteString s1) { - return s0.concat(s1); - } - - /** SQL {@code RTRIM} function applied to string. */ - public static String rtrim(String s) { - return trim_(s, false, true, ' '); - } - - /** SQL {@code LTRIM} function. */ - public static String ltrim(String s) { - return trim_(s, true, false, ' '); - } - - /** SQL {@code TRIM(... seek FROM s)} function. */ - public static String trim(boolean leading, boolean trailing, String seek, - String s) { - return trim_(s, leading, trailing, seek.charAt(0)); - } - - /** SQL {@code TRIM} function. */ - private static String trim_(String s, boolean left, boolean right, char c) { - int j = s.length(); - if (right) { - for (;;) { - if (j == 0) { - return ""; - } - if (s.charAt(j - 1) != c) { - break; - } - --j; - } - } - int i = 0; - if (left) { - for (;;) { - if (i == j) { - return ""; - } - if (s.charAt(i) != c) { - break; - } - ++i; - } - } - return s.substring(i, j); - } - - /** SQL {@code TRIM} function applied to binary string. */ - public static ByteString trim(ByteString s) { - return trim_(s, true, true); - } - - /** Helper for CAST. */ - public static ByteString rtrim(ByteString s) { - return trim_(s, false, true); - } - - /** SQL {@code TRIM} function applied to binary string. */ - private static ByteString trim_(ByteString s, boolean left, boolean right) { - int j = s.length(); - if (right) { - for (;;) { - if (j == 0) { - return ByteString.EMPTY; - } - if (s.byteAt(j - 1) != 0) { - break; - } - --j; - } - } - int i = 0; - if (left) { - for (;;) { - if (i == j) { - return ByteString.EMPTY; - } - if (s.byteAt(i) != 0) { - break; - } - ++i; - } - } - return s.substring(i, j); - } - - /** SQL {@code OVERLAY} function. */ - public static String overlay(String s, String r, int start) { - if (s == null || r == null) { - return null; - } - return s.substring(0, start - 1) - + r - + s.substring(start - 1 + r.length()); - } - - /** SQL {@code OVERLAY} function. */ - public static String overlay(String s, String r, int start, int length) { - if (s == null || r == null) { - return null; - } - return s.substring(0, start - 1) - + r - + s.substring(start - 1 + length); - } - - /** SQL {@code OVERLAY} function applied to binary strings. */ - public static ByteString overlay(ByteString s, ByteString r, int start) { - if (s == null || r == null) { - return null; - } - return s.substring(0, start - 1) - .concat(r) - .concat(s.substring(start - 1 + r.length())); - } - - /** SQL {@code OVERLAY} function applied to binary strings. */ - public static ByteString overlay(ByteString s, ByteString r, int start, - int length) { - if (s == null || r == null) { - return null; - } - return s.substring(0, start - 1) - .concat(r) - .concat(s.substring(start - 1 + length)); - } - - /** SQL {@code LIKE} function. */ - public static boolean like(String s, String pattern) { - final String regex = Like.sqlToRegexLike(pattern, null); - return Pattern.matches(regex, s); - } - - /** SQL {@code LIKE} function with escape. */ - public static boolean like(String s, String pattern, String escape) { - final String regex = Like.sqlToRegexLike(pattern, escape); - return Pattern.matches(regex, s); - } - - /** SQL {@code SIMILAR} function. */ - public static boolean similar(String s, String pattern) { - final String regex = Like.sqlToRegexSimilar(pattern, null); - return Pattern.matches(regex, s); - } - - /** SQL {@code SIMILAR} function with escape. */ - public static boolean similar(String s, String pattern, String escape) { - final String regex = Like.sqlToRegexSimilar(pattern, escape); - return Pattern.matches(regex, s); - } - - // = - - /** SQL = operator applied to Object values (including String; neither - * side may be null). */ - public static boolean eq(Object b0, Object b1) { - return b0.equals(b1); - } - - /** SQL = operator applied to BigDecimal values (neither may be null). */ - public static boolean eq(BigDecimal b0, BigDecimal b1) { - return b0.stripTrailingZeros().equals(b1.stripTrailingZeros()); - } - - // <> - - /** SQL <> operator applied to Object values (including String; - * neither side may be null). */ - public static boolean ne(Object b0, Object b1) { - return !b0.equals(b1); - } - - /** SQL <> operator applied to BigDecimal values. */ - public static boolean ne(BigDecimal b0, BigDecimal b1) { - return b0.compareTo(b1) != 0; - } - - // < - - /** SQL < operator applied to boolean values. */ - public static boolean lt(boolean b0, boolean b1) { - return compare(b0, b1) < 0; - } - - /** SQL < operator applied to String values. */ - public static boolean lt(String b0, String b1) { - return b0.compareTo(b1) < 0; - } - - /** SQL < operator applied to ByteString values. */ - public static boolean lt(ByteString b0, ByteString b1) { - return b0.compareTo(b1) < 0; - } - - /** SQL < operator applied to BigDecimal values. */ - public static boolean lt(BigDecimal b0, BigDecimal b1) { - return b0.compareTo(b1) < 0; - } - - // <= - - /** SQL ≤ operator applied to boolean values. */ - public static boolean le(boolean b0, boolean b1) { - return compare(b0, b1) <= 0; - } - - /** SQL ≤ operator applied to String values. */ - public static boolean le(String b0, String b1) { - return b0.compareTo(b1) <= 0; - } - - /** SQL ≤ operator applied to ByteString values. */ - public static boolean le(ByteString b0, ByteString b1) { - return b0.compareTo(b1) <= 0; - } - - /** SQL ≤ operator applied to BigDecimal values. */ - public static boolean le(BigDecimal b0, BigDecimal b1) { - return b0.compareTo(b1) <= 0; - } - - // > - - /** SQL > operator applied to boolean values. */ - public static boolean gt(boolean b0, boolean b1) { - return compare(b0, b1) > 0; - } - - /** SQL > operator applied to String values. */ - public static boolean gt(String b0, String b1) { - return b0.compareTo(b1) > 0; - } - - /** SQL > operator applied to ByteString values. */ - public static boolean gt(ByteString b0, ByteString b1) { - return b0.compareTo(b1) > 0; - } - - /** SQL > operator applied to BigDecimal values. */ - public static boolean gt(BigDecimal b0, BigDecimal b1) { - return b0.compareTo(b1) > 0; - } - - // >= - - /** SQL ≥ operator applied to boolean values. */ - public static boolean ge(boolean b0, boolean b1) { - return compare(b0, b1) >= 0; - } - - /** SQL ≥ operator applied to String values. */ - public static boolean ge(String b0, String b1) { - return b0.compareTo(b1) >= 0; - } - - /** SQL ≥ operator applied to ByteString values. */ - public static boolean ge(ByteString b0, ByteString b1) { - return b0.compareTo(b1) >= 0; - } - - /** SQL ≥ operator applied to BigDecimal values. */ - public static boolean ge(BigDecimal b0, BigDecimal b1) { - return b0.compareTo(b1) >= 0; - } - - // + - - /** SQL + operator applied to int values. */ - public static int plus(int b0, int b1) { - return b0 + b1; - } - - /** SQL + operator applied to int values; left side may be - * null. */ - public static Integer plus(Integer b0, int b1) { - return b0 == null ? null : (b0 + b1); - } - - /** SQL + operator applied to int values; right side may be - * null. */ - public static Integer plus(int b0, Integer b1) { - return b1 == null ? null : (b0 + b1); - } - - /** SQL + operator applied to nullable int values. */ - public static Integer plus(Integer b0, Integer b1) { - return (b0 == null || b1 == null) ? null : (b0 + b1); - } - - /** SQL + operator applied to nullable long and int values. */ - public static Long plus(Long b0, Integer b1) { - return (b0 == null || b1 == null) - ? null - : (b0.longValue() + b1.longValue()); - } - - /** SQL + operator applied to nullable int and long values. */ - public static Long plus(Integer b0, Long b1) { - return (b0 == null || b1 == null) - ? null - : (b0.longValue() + b1.longValue()); - } - - /** SQL + operator applied to BigDecimal values. */ - public static BigDecimal plus(BigDecimal b0, BigDecimal b1) { - return (b0 == null || b1 == null) ? null : b0.add(b1); - } - - // - - - /** SQL - operator applied to int values. */ - public static int minus(int b0, int b1) { - return b0 - b1; - } - - /** SQL - operator applied to int values; left side may be - * null. */ - public static Integer minus(Integer b0, int b1) { - return b0 == null ? null : (b0 - b1); - } - - /** SQL - operator applied to int values; right side may be - * null. */ - public static Integer minus(int b0, Integer b1) { - return b1 == null ? null : (b0 - b1); - } - - /** SQL - operator applied to nullable int values. */ - public static Integer minus(Integer b0, Integer b1) { - return (b0 == null || b1 == null) ? null : (b0 - b1); - } - - /** SQL - operator applied to nullable long and int values. */ - public static Long minus(Long b0, Integer b1) { - return (b0 == null || b1 == null) - ? null - : (b0.longValue() - b1.longValue()); - } - - /** SQL - operator applied to nullable int and long values. */ - public static Long minus(Integer b0, Long b1) { - return (b0 == null || b1 == null) - ? null - : (b0.longValue() - b1.longValue()); - } - - /** SQL - operator applied to BigDecimal values. */ - public static BigDecimal minus(BigDecimal b0, BigDecimal b1) { - return (b0 == null || b1 == null) ? null : b0.subtract(b1); - } - - // / - - /** SQL / operator applied to int values. */ - public static int divide(int b0, int b1) { - return b0 / b1; - } - - /** SQL / operator applied to int values; left side may be - * null. */ - public static Integer divide(Integer b0, int b1) { - return b0 == null ? null : (b0 / b1); - } - - /** SQL / operator applied to int values; right side may be - * null. */ - public static Integer divide(int b0, Integer b1) { - return b1 == null ? null : (b0 / b1); - } - - /** SQL / operator applied to nullable int values. */ - public static Integer divide(Integer b0, Integer b1) { - return (b0 == null || b1 == null) ? null : (b0 / b1); - } - - /** SQL / operator applied to nullable long and int values. */ - public static Long divide(Long b0, Integer b1) { - return (b0 == null || b1 == null) - ? null - : (b0.longValue() / b1.longValue()); - } - - /** SQL / operator applied to nullable int and long values. */ - public static Long divide(Integer b0, Long b1) { - return (b0 == null || b1 == null) - ? null - : (b0.longValue() / b1.longValue()); - } - - /** SQL / operator applied to BigDecimal values. */ - public static BigDecimal divide(BigDecimal b0, BigDecimal b1) { - // OVERRIDE POINT - return (b0 == null || b1 == null) ? null : b0.divide(b1, MathContext.DECIMAL64); - } - - // * - - /** SQL * operator applied to int values. */ - public static int multiply(int b0, int b1) { - return b0 * b1; - } - - /** SQL * operator applied to int values; left side may be - * null. */ - public static Integer multiply(Integer b0, int b1) { - return b0 == null ? null : (b0 * b1); - } - - /** SQL * operator applied to int values; right side may be - * null. */ - public static Integer multiply(int b0, Integer b1) { - return b1 == null ? null : (b0 * b1); - } - - /** SQL * operator applied to nullable int values. */ - public static Integer multiply(Integer b0, Integer b1) { - return (b0 == null || b1 == null) ? null : (b0 * b1); - } - - /** SQL * operator applied to nullable long and int values. */ - public static Long multiply(Long b0, Integer b1) { - return (b0 == null || b1 == null) - ? null - : (b0.longValue() * b1.longValue()); - } - - /** SQL * operator applied to nullable int and long values. */ - public static Long multiply(Integer b0, Long b1) { - return (b0 == null || b1 == null) - ? null - : (b0.longValue() * b1.longValue()); - } - - /** SQL * operator applied to BigDecimal values. */ - public static BigDecimal multiply(BigDecimal b0, BigDecimal b1) { - return (b0 == null || b1 == null) ? null : b0.multiply(b1); - } - - // EXP - - /** SQL EXP operator applied to double values. */ - public static double exp(double b0) { - return Math.exp(b0); - } - - public static double exp(long b0) { - return Math.exp(b0); - } - - // POWER - - /** SQL POWER operator applied to double values. */ - public static double power(double b0, double b1) { - return Math.pow(b0, b1); - } - - public static double power(long b0, long b1) { - return Math.pow(b0, b1); - } - - public static double power(long b0, BigDecimal b1) { - return Math.pow(b0, b1.doubleValue()); - } - - // LN - - /** SQL {@code LN(number)} function applied to double values. */ - public static double ln(double d) { - return Math.log(d); - } - - /** SQL {@code LN(number)} function applied to long values. */ - public static double ln(long b0) { - return Math.log(b0); - } - - /** SQL {@code LN(number)} function applied to BigDecimal values. */ - public static double ln(BigDecimal d) { - return Math.log(d.doubleValue()); - } - - // LOG10 - - /** SQL LOG10(numeric) operator applied to double values. */ - public static double log10(double b0) { - return Math.log10(b0); - } - - /** SQL {@code LOG10(number)} function applied to long values. */ - public static double log10(long b0) { - return Math.log10(b0); - } - - /** SQL {@code LOG10(number)} function applied to BigDecimal values. */ - public static double log10(BigDecimal d) { - return Math.log10(d.doubleValue()); - } - - // MOD - - /** SQL MOD operator applied to byte values. */ - public static byte mod(byte b0, byte b1) { - return (byte) (b0 % b1); - } - - /** SQL MOD operator applied to short values. */ - public static short mod(short b0, short b1) { - return (short) (b0 % b1); - } - - /** SQL MOD operator applied to int values. */ - public static int mod(int b0, int b1) { - return b0 % b1; - } - - /** SQL MOD operator applied to long values. */ - public static long mod(long b0, long b1) { - return b0 % b1; - } - - // temporary - public static BigDecimal mod(BigDecimal b0, int b1) { - return mod(b0, BigDecimal.valueOf(b1)); - } - - // temporary - public static int mod(int b0, BigDecimal b1) { - return mod(b0, b1.intValue()); - } - - public static BigDecimal mod(BigDecimal b0, BigDecimal b1) { - final BigDecimal[] bigDecimals = b0.divideAndRemainder(b1); - return bigDecimals[1]; - } - - // ABS - - /** SQL ABS operator applied to byte values. */ - public static byte abs(byte b0) { - return (byte) Math.abs(b0); - } - - /** SQL ABS operator applied to short values. */ - public static short abs(short b0) { - return (short) Math.abs(b0); - } - - /** SQL ABS operator applied to int values. */ - public static int abs(int b0) { - return Math.abs(b0); - } - - /** SQL ABS operator applied to long values. */ - public static long abs(long b0) { - return Math.abs(b0); - } - - /** SQL ABS operator applied to float values. */ - public static float abs(float b0) { - return Math.abs(b0); - } - - /** SQL ABS operator applied to double values. */ - public static double abs(double b0) { - return Math.abs(b0); - } - - /** SQL ABS operator applied to BigDecimal values. */ - public static BigDecimal abs(BigDecimal b0) { - return b0.abs(); - } - - // Helpers - - /** Helper for implementing MIN. Somewhat similar to LEAST operator. */ - public static > T lesser(T b0, T b1) { - return b0 == null || b0.compareTo(b1) > 0 ? b1 : b0; - } - - /** LEAST operator. */ - public static > T least(T b0, T b1) { - return b0 == null || b1 != null && b0.compareTo(b1) > 0 ? b1 : b0; - } - - public static boolean greater(boolean b0, boolean b1) { - return b0 || b1; - } - - public static boolean lesser(boolean b0, boolean b1) { - return b0 && b1; - } - - public static byte greater(byte b0, byte b1) { - return b0 > b1 ? b0 : b1; - } - - public static byte lesser(byte b0, byte b1) { - return b0 > b1 ? b1 : b0; - } - - public static char greater(char b0, char b1) { - return b0 > b1 ? b0 : b1; - } - - public static char lesser(char b0, char b1) { - return b0 > b1 ? b1 : b0; - } - - public static short greater(short b0, short b1) { - return b0 > b1 ? b0 : b1; - } - - public static short lesser(short b0, short b1) { - return b0 > b1 ? b1 : b0; - } - - public static int greater(int b0, int b1) { - return b0 > b1 ? b0 : b1; - } - - public static int lesser(int b0, int b1) { - return b0 > b1 ? b1 : b0; - } - - public static long greater(long b0, long b1) { - return b0 > b1 ? b0 : b1; - } - - public static long lesser(long b0, long b1) { - return b0 > b1 ? b1 : b0; - } - - public static float greater(float b0, float b1) { - return b0 > b1 ? b0 : b1; - } - - public static float lesser(float b0, float b1) { - return b0 > b1 ? b1 : b0; - } - - public static double greater(double b0, double b1) { - return b0 > b1 ? b0 : b1; - } - - public static double lesser(double b0, double b1) { - return b0 > b1 ? b1 : b0; - } - - /** Helper for implementing MAX. Somewhat similar to GREATEST operator. */ - public static > T greater(T b0, T b1) { - return b0 == null || b0.compareTo(b1) < 0 ? b1 : b0; - } - - /** GREATEST operator. */ - public static > T greatest(T b0, T b1) { - return b0 == null || b1 != null && b0.compareTo(b1) < 0 ? b1 : b0; - } - - /** Boolean comparison. */ - public static int compare(boolean x, boolean y) { - return x == y ? 0 : x ? 1 : -1; - } - - /** CAST(FLOAT AS VARCHAR). */ - public static String toString(float x) { - if (x == 0) { - return "0E0"; - } - BigDecimal bigDecimal = - new BigDecimal(x, MathContext.DECIMAL32).stripTrailingZeros(); - final String s = bigDecimal.toString(); - return s.replaceAll("0*E", "E").replace("E+", "E"); - } - - /** CAST(DOUBLE AS VARCHAR). */ - public static String toString(double x) { - if (x == 0) { - return "0E0"; - } - BigDecimal bigDecimal = - new BigDecimal(x, MathContext.DECIMAL64).stripTrailingZeros(); - final String s = bigDecimal.toString(); - return s.replaceAll("0*E", "E").replace("E+", "E"); - } - - /** CAST(DECIMAL AS VARCHAR). */ - public static String toString(BigDecimal x) { - final String s = x.toString(); - if (s.startsWith("0")) { - // we want ".1" not "0.1" - return s.substring(1); - } else if (s.startsWith("-0")) { - // we want "-.1" not "-0.1" - return "-" + s.substring(2); - } else { - return s; - } - } - - /** CAST(BOOLEAN AS VARCHAR). */ - public static String toString(boolean x) { - // Boolean.toString returns lower case -- no good. - return x ? "TRUE" : "FALSE"; - } - - @NonDeterministic - private static Object cannotConvert(Object o, Class toType) { - throw new RuntimeException("Cannot convert " + o + " to " + toType); - } - - /** CAST(VARCHAR AS BOOLEAN). */ - public static boolean toBoolean(String s) { - s = trim_(s, true, true, ' '); - if (s.equalsIgnoreCase("TRUE")) { - return true; - } else if (s.equalsIgnoreCase("FALSE")) { - return false; - } else { - throw new RuntimeException("Invalid character for cast"); - } - } - - public static boolean toBoolean(Number number) { - return !number.equals(0); - } - - public static boolean toBoolean(Object o) { - return o instanceof Boolean ? (Boolean) o - : o instanceof Number ? toBoolean((Number) o) - : o instanceof String ? toBoolean((String) o) - : (Boolean) cannotConvert(o, boolean.class); - } - - // Don't need parseByte etc. - Byte.parseByte is sufficient. - - public static byte toByte(Object o) { - return o instanceof Byte ? (Byte) o - : o instanceof Number ? toByte((Number) o) - : Byte.parseByte(o.toString()); - } - - public static byte toByte(Number number) { - return number.byteValue(); - } - - public static char toChar(String s) { - return s.charAt(0); - } - - public static Character toCharBoxed(String s) { - return s.charAt(0); - } - - public static short toShort(String s) { - return Short.parseShort(s.trim()); - } - - public static short toShort(Number number) { - return number.shortValue(); - } - - public static short toShort(Object o) { - return o instanceof Short ? (Short) o - : o instanceof Number ? toShort((Number) o) - : o instanceof String ? toShort((String) o) - : (Short) cannotConvert(o, short.class); - } - - public static int toInt(java.util.Date v) { - return toInt(v, LOCAL_TZ); - } - - public static int toInt(java.util.Date v, TimeZone timeZone) { - return (int) (toLong(v, timeZone) / DateTimeUtil.MILLIS_PER_DAY); - } - - public static Integer toIntOptional(java.util.Date v) { - return v == null ? null : toInt(v); - } - - public static Integer toIntOptional(java.util.Date v, TimeZone timeZone) { - return v == null - ? null - : toInt(v, timeZone); - } - - public static long toLong(Date v) { - return toLong(v, LOCAL_TZ); - } - - public static int toInt(java.sql.Time v) { - return (int) (toLong(v) % DateTimeUtil.MILLIS_PER_DAY); - } - - public static Integer toIntOptional(java.sql.Time v) { - return v == null ? null : toInt(v); - } - - public static int toInt(String s) { - return Integer.parseInt(s.trim()); - } - - public static int toInt(Number number) { - return number.intValue(); - } - - public static int toInt(Object o) { - return o instanceof Integer ? (Integer) o - : o instanceof Number ? toInt((Number) o) - : o instanceof String ? toInt((String) o) - : (Integer) cannotConvert(o, int.class); - } - - public static long toLong(Timestamp v) { - return toLong(v, LOCAL_TZ); - } - - // mainly intended for java.sql.Timestamp but works for other dates also - public static long toLong(java.util.Date v, TimeZone timeZone) { - final long time = v.getTime(); - return time + timeZone.getOffset(time); - } - - // mainly intended for java.sql.Timestamp but works for other dates also - public static Long toLongOptional(java.util.Date v) { - return v == null ? null : toLong(v, LOCAL_TZ); - } - - public static Long toLongOptional(Timestamp v, TimeZone timeZone) { - if (v == null) { - return null; - } - return toLong(v, LOCAL_TZ); - } - - public static long toLong(String s) { - if (s.startsWith("199") && s.contains(":")) { - return Timestamp.valueOf(s).getTime(); - } - return Long.parseLong(s.trim()); - } - - public static long toLong(Number number) { - return number.longValue(); - } - - public static long toLong(Object o) { - return o instanceof Long ? (Long) o - : o instanceof Number ? toLong((Number) o) - : o instanceof String ? toLong((String) o) - : (Long) cannotConvert(o, long.class); - } - - public static float toFloat(String s) { - return Float.parseFloat(s.trim()); - } - - public static float toFloat(Number number) { - return number.floatValue(); - } - - public static float toFloat(Object o) { - return o instanceof Float ? (Float) o - : o instanceof Number ? toFloat((Number) o) - : o instanceof String ? toFloat((String) o) - : (Float) cannotConvert(o, float.class); - } - - public static double toDouble(String s) { - return Double.parseDouble(s.trim()); - } - - public static double toDouble(Number number) { - return number.doubleValue(); - } - - public static double toDouble(Object o) { - return o instanceof Double ? (Double) o - : o instanceof Number ? toDouble((Number) o) - : o instanceof String ? toDouble((String) o) - : (Double) cannotConvert(o, double.class); - } - - public static BigDecimal toBigDecimal(String s) { - return new BigDecimal(s.trim()); - } - - public static BigDecimal toBigDecimal(Number number) { - // There are some values of "long" that cannot be represented as "double". - // Not so "int". If it isn't a long, go straight to double. - return number instanceof BigDecimal ? (BigDecimal) number - : number instanceof BigInteger ? new BigDecimal((BigInteger) number) - : number instanceof Long ? new BigDecimal(number.longValue()) - : new BigDecimal(number.doubleValue()); - } - - public static BigDecimal toBigDecimal(Object o) { - return o instanceof Number ? toBigDecimal((Number) o) - : toBigDecimal(o.toString()); - } - - // Don't need shortValueOf etc. - Short.valueOf is sufficient. - - /** Helper for CAST(... AS VARCHAR(maxLength)). */ - public static String truncate(String s, int maxLength) { - return s == null ? null - : s.length() > maxLength ? s.substring(0, maxLength) - : s; - } - - /** Helper for CAST(... AS VARBINARY(maxLength)). */ - public static ByteString truncate(ByteString s, int maxLength) { - return s == null ? null - : s.length() > maxLength ? s.substring(0, maxLength) - : s; - } - - /** SQL {@code POSITION(seek IN string)} function. */ - public static int position(String seek, String s) { - return s.indexOf(seek) + 1; - } - - /** SQL {@code POSITION(seek IN string)} function. */ - public static int position(ByteString seek, ByteString s) { - return s.indexOf(seek) + 1; - } - - /** Cheap, unsafe, long power. power(2, 3) returns 8. */ - public static long powerX(long a, long b) { - long x = 1; - while (b > 0) { - x *= a; - --b; - } - return x; - } - - /** Helper for rounding. Truncate(12345, 1000) returns 12000. */ - public static long round(long v, long x) { - return truncate(v + x / 2, x); - } - - /** Helper for rounding. Truncate(12345, 1000) returns 12000. */ - public static long truncate(long v, long x) { - long remainder = v % x; - if (remainder < 0) { - remainder += x; - } - return v - remainder; - } - - /** Helper for rounding. Truncate(12345, 1000) returns 12000. */ - public static int round(int v, int x) { - return truncate(v + x / 2, x); - } - - /** Helper for rounding. Truncate(12345, 1000) returns 12000. */ - public static int truncate(int v, int x) { - int remainder = v % x; - if (remainder < 0) { - remainder += x; - } - return v - remainder; - } - - /** Helper for CAST({timestamp} AS VARCHAR(n)). */ - public static String unixTimestampToString(long timestamp) { - final StringBuilder buf = new StringBuilder(17); - int date = (int) (timestamp / DateTimeUtil.MILLIS_PER_DAY); - int time = (int) (timestamp % DateTimeUtil.MILLIS_PER_DAY); - if (time < 0) { - --date; - time += DateTimeUtil.MILLIS_PER_DAY; - } - unixDateToString(buf, date); - buf.append(' '); - unixTimeToString(buf, time); - return buf.toString(); - } - - /** Helper for CAST({timestamp} AS VARCHAR(n)). */ - public static String unixTimeToString(int time) { - final StringBuilder buf = new StringBuilder(8); - unixTimeToString(buf, time); - return buf.toString(); - } - - private static void unixTimeToString(StringBuilder buf, int time) { - int h = time / 3600000; - int time2 = time % 3600000; - int m = time2 / 60000; - int time3 = time2 % 60000; - int s = time3 / 1000; - int ms = time3 % 1000; - int2(buf, h); - buf.append(':'); - int2(buf, m); - buf.append(':'); - int2(buf, s); - } - - /** SQL {@code CURRENT_TIMESTAMP} function. */ - @NonDeterministic - public static long currentTimestamp(DataContext root) { - // Cast required for JDK 1.6. - return (Long) DataContext.Variable.CURRENT_TIMESTAMP.get(root); - } - - /** SQL {@code CURRENT_TIME} function. */ - @NonDeterministic - public static int currentTime(DataContext root) { - int time = (int) (currentTimestamp(root) % DateTimeUtil.MILLIS_PER_DAY); - if (time < 0) { - time += DateTimeUtil.MILLIS_PER_DAY; - } - return time; - } - - /** SQL {@code CURRENT_DATE} function. */ - @NonDeterministic - public static int currentDate(DataContext root) { - final long timestamp = currentTimestamp(root); - int date = (int) (timestamp / DateTimeUtil.MILLIS_PER_DAY); - final int time = (int) (timestamp % DateTimeUtil.MILLIS_PER_DAY); - if (time < 0) { - --date; - } - return date; - } - - /** SQL {@code LOCAL_TIMESTAMP} function. */ - @NonDeterministic - public static long localTimestamp(DataContext root) { - // Cast required for JDK 1.6. - return (Long) DataContext.Variable.LOCAL_TIMESTAMP.get(root); - } - - /** SQL {@code LOCAL_TIME} function. */ - @NonDeterministic - public static int localTime(DataContext root) { - return (int) (localTimestamp(root) % DateTimeUtil.MILLIS_PER_DAY); - } - - private static void int2(StringBuilder buf, int i) { - buf.append((char) ('0' + (i / 10) % 10)); - buf.append((char) ('0' + i % 10)); - } - - private static void int4(StringBuilder buf, int i) { - buf.append((char) ('0' + (i / 1000) % 10)); - buf.append((char) ('0' + (i / 100) % 10)); - buf.append((char) ('0' + (i / 10) % 10)); - buf.append((char) ('0' + i % 10)); - } - - public static int dateStringToUnixDate(String s) { - int hyphen1 = s.indexOf('-'); - int y; - int m; - int d; - if (hyphen1 < 0) { - y = Integer.parseInt(s.trim()); - m = 1; - d = 1; - } else { - y = Integer.parseInt(s.substring(0, hyphen1).trim()); - final int hyphen2 = s.indexOf('-', hyphen1 + 1); - if (hyphen2 < 0) { - m = Integer.parseInt(s.substring(hyphen1 + 1).trim()); - d = 1; - } else { - m = Integer.parseInt(s.substring(hyphen1 + 1, hyphen2).trim()); - d = Integer.parseInt(s.substring(hyphen2 + 1).trim()); - } - } - return ymdToUnixDate(y, m, d); - } - - public static int timeStringToUnixDate(String v) { - return timeStringToUnixDate(v, 0); - } - - public static int timeStringToUnixDate(String v, int start) { - final int colon1 = v.indexOf(':', start); - int hour; - int minute; - int second; - int milli; - if (colon1 < 0) { - hour = Integer.parseInt(v.trim()); - minute = 1; - second = 1; - milli = 0; - } else { - hour = Integer.parseInt(v.substring(start, colon1).trim()); - final int colon2 = v.indexOf(':', colon1 + 1); - if (colon2 < 0) { - minute = Integer.parseInt(v.substring(colon1 + 1).trim()); - second = 1; - milli = 0; - } else { - minute = Integer.parseInt(v.substring(colon1 + 1, colon2).trim()); - int dot = v.indexOf('.', colon2); - if (dot < 0) { - second = Integer.parseInt(v.substring(colon2 + 1).trim()); - milli = 0; - } else { - second = Integer.parseInt(v.substring(colon2 + 1, dot).trim()); - milli = Integer.parseInt(v.substring(dot + 1).trim()); - } - } - } - return hour * (int) DateTimeUtil.MILLIS_PER_HOUR - + minute * (int) DateTimeUtil.MILLIS_PER_MINUTE - + second * (int) DateTimeUtil.MILLIS_PER_SECOND - + milli; - } - - public static long timestampStringToUnixDate(String s) { - final long d; - final long t; - s = s.trim(); - int space = s.indexOf(' '); - if (space >= 0) { - d = dateStringToUnixDate(s.substring(0, space)); - t = timeStringToUnixDate(s, space + 1); - } else { - d = dateStringToUnixDate(s); - t = 0; - } - return d * DateTimeUtil.MILLIS_PER_DAY + t; - } - - /** Helper for CAST({date} AS VARCHAR(n)). */ - public static String unixDateToString(int date) { - final StringBuilder buf = new StringBuilder(10); - unixDateToString(buf, date); - return buf.toString(); - } - - private static void unixDateToString(StringBuilder buf, int date) { - julianToString(buf, date + EPOCH_JULIAN); - } - - private static void julianToString(StringBuilder buf, int julian) { - // this shifts the epoch back to astronomical year -4800 instead of the - // start of the Christian era in year AD 1 of the proleptic Gregorian - // calendar. - int j = julian + 32044; - int g = j / 146097; - int dg = j % 146097; - int c = (dg / 36524 + 1) * 3 / 4; - int dc = dg - c * 36524; - int b = dc / 1461; - int db = dc % 1461; - int a = (db / 365 + 1) * 3 / 4; - int da = db - a * 365; - - // integer number of full years elapsed since March 1, 4801 BC - int y = g * 400 + c * 100 + b * 4 + a; - // integer number of full months elapsed since the last March 1 - int m = (da * 5 + 308) / 153 - 2; - // number of days elapsed since day 1 of the month - int d = da - (m + 4) * 153 / 5 + 122; - int year = y - 4800 + (m + 2) / 12; - int month = (m + 2) % 12 + 1; - int day = d + 1; - int4(buf, year); - buf.append('-'); - int2(buf, month); - buf.append('-'); - int2(buf, day); - } - - public static long unixDateExtract(TimeUnitRange range, long date) { - return julianExtract(range, (int) date + EPOCH_JULIAN); - } - - private static int julianExtract(TimeUnitRange range, int julian) { - // this shifts the epoch back to astronomical year -4800 instead of the - // start of the Christian era in year AD 1 of the proleptic Gregorian - // calendar. - int j = julian + 32044; - int g = j / 146097; - int dg = j % 146097; - int c = (dg / 36524 + 1) * 3 / 4; - int dc = dg - c * 36524; - int b = dc / 1461; - int db = dc % 1461; - int a = (db / 365 + 1) * 3 / 4; - int da = db - a * 365; - - // integer number of full years elapsed since March 1, 4801 BC - int y = g * 400 + c * 100 + b * 4 + a; - // integer number of full months elapsed since the last March 1 - int m = (da * 5 + 308) / 153 - 2; - // number of days elapsed since day 1 of the month - int d = da - (m + 4) * 153 / 5 + 122; - int year = y - 4800 + (m + 2) / 12; - int month = (m + 2) % 12 + 1; - int day = d + 1; - switch (range) { - case YEAR: - return year; - case MONTH: - return month; - case DAY: - return day; - default: - throw new AssertionError(range); - } - } - - public static int ymdToUnixDate(int year, int month, int day) { - final int julian = ymdToJulian(year, month, day); - return julian - EPOCH_JULIAN; - } - - public static int ymdToJulian(int year, int month, int day) { - int a = (14 - month) / 12; - int y = year + 4800 - a; - int m = month + 12 * a - 3; - int j = day + (153 * m + 2) / 5 - + 365 * y - + y / 4 - - y / 100 - + y / 400 - - 32045; - if (j < 2299161) { - j = day + (153 * m + 2) / 5 + 365 * y + y / 4 - 32083; - } - return j; - } - - public static String intervalYearMonthToString(int v, TimeUnitRange range) { - final StringBuilder buf = new StringBuilder(); - if (v >= 0) { - buf.append('+'); - } else { - buf.append('-'); - v = -v; - } - final int y; - final int m; - switch (range) { - case YEAR: - v = roundUp(v, 12); - y = v / 12; - buf.append(y); - break; - case YEAR_TO_MONTH: - y = v / 12; - buf.append(y); - buf.append('-'); - m = v % 12; - number(buf, m, 2); - break; - case MONTH: - m = v; - buf.append(m); - break; - default: - throw new AssertionError(range); - } - return buf.toString(); - } - - private static StringBuilder number(StringBuilder buf, int v, int n) { - for (int k = digitCount(v); k < n; k++) { - buf.append('0'); - } - return buf.append(v); - } - - public static int digitCount(int v) { - for (int n = 1;; n++) { - v /= 10; - if (v == 0) { - return n; - } - } - } - - public static String intervalDayTimeToString(long v, TimeUnitRange range, - int scale) { - final StringBuilder buf = new StringBuilder(); - if (v >= 0) { - buf.append('+'); - } else { - buf.append('-'); - v = -v; - } - final long ms; - final long s; - final long m; - final long h; - final long d; - switch (range) { - case DAY_TO_SECOND: - v = roundUp(v, powerX(10, 3 - scale)); - ms = v % 1000; - v /= 1000; - s = v % 60; - v /= 60; - m = v % 60; - v /= 60; - h = v % 24; - v /= 24; - d = v; - buf.append((int) d); - buf.append(' '); - number(buf, (int) h, 2); - buf.append(':'); - number(buf, (int) m, 2); - buf.append(':'); - number(buf, (int) s, 2); - fraction(buf, scale, ms); - break; - case DAY_TO_MINUTE: - v = roundUp(v, 1000 * 60); - v /= 1000; - v /= 60; - m = v % 60; - v /= 60; - h = v % 24; - v /= 24; - d = v; - buf.append((int) d); - buf.append(' '); - number(buf, (int) h, 2); - buf.append(':'); - number(buf, (int) m, 2); - break; - case DAY_TO_HOUR: - v = roundUp(v, 1000 * 60 * 60); - v /= 1000; - v /= 60; - v /= 60; - h = v % 24; - v /= 24; - d = v; - buf.append((int) d); - buf.append(' '); - number(buf, (int) h, 2); - break; - case DAY: - v = roundUp(v, 1000 * 60 * 60 * 24); - d = v / (1000 * 60 * 60 * 24); - buf.append((int) d); - break; - case HOUR: - v = roundUp(v, 1000 * 60 * 60); - v /= 1000; - v /= 60; - v /= 60; - h = v; - buf.append((int) h); - break; - case HOUR_TO_MINUTE: - v = roundUp(v, 1000 * 60); - v /= 1000; - v /= 60; - m = v % 60; - v /= 60; - h = v; - buf.append((int) h); - buf.append(':'); - number(buf, (int) m, 2); - break; - case HOUR_TO_SECOND: - v = roundUp(v, powerX(10, 3 - scale)); - ms = v % 1000; - v /= 1000; - s = v % 60; - v /= 60; - m = v % 60; - v /= 60; - h = v; - buf.append((int) h); - buf.append(':'); - number(buf, (int) m, 2); - buf.append(':'); - number(buf, (int) s, 2); - fraction(buf, scale, ms); - break; - case MINUTE_TO_SECOND: - v = roundUp(v, powerX(10, 3 - scale)); - ms = v % 1000; - v /= 1000; - s = v % 60; - v /= 60; - m = v; - buf.append((int) m); - buf.append(':'); - number(buf, (int) s, 2); - fraction(buf, scale, ms); - break; - case MINUTE: - v = roundUp(v, 1000 * 60); - v /= 1000; - v /= 60; - m = v; - buf.append((int) m); - break; - case SECOND: - v = roundUp(v, powerX(10, 3 - scale)); - ms = v % 1000; - v /= 1000; - s = v; - buf.append((int) s); - fraction(buf, scale, ms); - break; - default: - throw new AssertionError(range); - } - return buf.toString(); - } - - /** - * Rounds a dividend to the nearest divisor. - * For example roundUp(31, 10) yields 30; roundUp(37, 10) yields 40. - * @param dividend Number to be divided - * @param divisor Number to divide by - * @return Rounded dividend - */ - private static long roundUp(long dividend, long divisor) { - long remainder = dividend % divisor; - dividend -= remainder; - if (remainder * 2 > divisor) { - dividend += divisor; - } - return dividend; - } - - private static int roundUp(int dividend, int divisor) { - int remainder = dividend % divisor; - dividend -= remainder; - if (remainder * 2 > divisor) { - dividend += divisor; - } - return dividend; - } - - private static void fraction(StringBuilder buf, int scale, long ms) { - if (scale > 0) { - buf.append('.'); - long v1 = scale == 3 ? ms - : scale == 2 ? ms / 10 - : scale == 1 ? ms / 100 - : 0; - number(buf, (int) v1, scale); - } - } - - /** Helper for "array element reference". Caller has already ensured that - * array and index are not null. Index is 1-based, per SQL. */ - public static Object arrayItem(List list, int item) { - if (item < 1 || item > list.size()) { - return null; - } - return list.get(item - 1); - } - - /** Helper for "map element reference". Caller has already ensured that - * array and index are not null. Index is 1-based, per SQL. */ - public static Object mapItem(Map map, Object item) { - return map.get(item); - } - - /** Implements the {@code [ ... ]} operator on an object whose type is not - * known until runtime. - */ - public static Object item(Object object, Object index) { - if (object instanceof Map) { - return ((Map) object).get(index); - } - if (object instanceof List && index instanceof Number) { - List list = (List) object; - return list.get(((Number) index).intValue()); - } - return null; - } - - /** NULL → FALSE, FALSE → FALSE, TRUE → TRUE. */ - public static boolean isTrue(Boolean b) { - return b != null && b; - } - - /** NULL → TRUE, FALSE → FALSE, TRUE → TRUE. */ - public static boolean isNotFalse(Boolean b) { - return b == null || b; - } - - /** NULL → NULL, FALSE → TRUE, TRUE → FALSE. */ - public static Boolean not(Boolean b) { - return (b == null) ? null : !b; - } - - /** Converts a JDBC array to a list. */ - public static List arrayToList(final java.sql.Array a) { - if (a == null) { - return null; - } - try { - return Primitive.asList(a.getArray()); - } catch (SQLException e) { - throw new RuntimeException(e); - } - } - - /** Support the SLICE function. */ - public static List slice(List list) { - return list; - } - - /** Support the ELEMENT function. */ - public static Object element(List list) { - switch (list.size()) { - case 0: - return null; - case 1: - return list.get(0); - default: - throw new RuntimeException("more than one value"); - } - } - - /** Returns a lambda that converts a list to an enumerable. */ - public static Function1, Enumerable> listToEnumerable() { - //noinspection unchecked - return (Function1, Enumerable>) (Function1) LIST_AS_ENUMERABLE; - } - - /** A range of time units. The first is more significant than the - * other (e.g. year-to-day) or the same as the other - * (e.g. month). */ - public enum TimeUnitRange { - YEAR, - YEAR_TO_MONTH, - MONTH, - DAY, - DAY_TO_HOUR, - DAY_TO_MINUTE, - DAY_TO_SECOND, - HOUR, - HOUR_TO_MINUTE, - HOUR_TO_SECOND, - MINUTE, - MINUTE_TO_SECOND, - SECOND; - - /** Whether this is in the YEAR-TO-MONTH family of intervals. */ - public boolean monthly() { - return ordinal() <= MONTH.ordinal(); - } - } -} - -// End SqlFunctions.java diff --git a/query/src/main/java/org/eigenbase/sql2rel/SqlToRelConverter.java b/query/src/main/java/org/eigenbase/sql2rel/SqlToRelConverter.java deleted file mode 100644 index 4855c93..0000000 --- a/query/src/main/java/org/eigenbase/sql2rel/SqlToRelConverter.java +++ /dev/null @@ -1,4782 +0,0 @@ -/* - * OVERRIDE POINTS: - * - getInSubqueryThreshold(), was `20`, now `Integer.MAX_VALUE` - * - isTrimUnusedFields(), override to false - */ - -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to you under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.eigenbase.sql2rel; - -import java.lang.reflect.Type; -import java.math.*; -import java.util.*; -import java.util.logging.*; - -import org.eigenbase.rel.*; -import org.eigenbase.rel.metadata.*; -import org.eigenbase.relopt.*; -import org.eigenbase.reltype.*; -import org.eigenbase.rex.*; -import org.eigenbase.sql.*; -import org.eigenbase.sql.fun.*; -import org.eigenbase.sql.parser.*; -import org.eigenbase.sql.type.*; -import org.eigenbase.sql.util.*; -import org.eigenbase.sql.validate.*; -import org.eigenbase.trace.*; -import org.eigenbase.util.*; -import org.eigenbase.util.mapping.Mappings; -import org.eigenbase.util14.*; - -import net.hydromatic.linq4j.Ord; -import net.hydromatic.optiq.ModifiableTable; -import net.hydromatic.optiq.TranslatableTable; -import net.hydromatic.optiq.prepare.Prepare; -import net.hydromatic.optiq.prepare.RelOptTableImpl; -import net.hydromatic.optiq.util.BitSets; - -import com.google.common.base.Function; -import com.google.common.collect.*; - -import static org.eigenbase.sql.SqlUtil.stripAs; -import static org.eigenbase.util.Static.RESOURCE; - -/** - * Converts a SQL parse tree (consisting of {@link org.eigenbase.sql.SqlNode} - * objects) into a relational algebra expression (consisting of - * {@link org.eigenbase.rel.RelNode} objects). - * - *

The public entry points are: {@link #convertQuery}, - * {@link #convertExpression(SqlNode)}. - */ -@SuppressWarnings({"unused", "rawtypes", "unchecked", "incomplete-switch", "deprecation"}) -public class SqlToRelConverter { - //~ Static fields/initializers --------------------------------------------- - - protected static final Logger SQL2REL_LOGGER = - EigenbaseTrace.getSqlToRelTracer(); - - private static final Function FN = - new Function() { - public SqlNode apply(SubQuery input) { - return input.node; - } - }; - - //~ Instance fields -------------------------------------------------------- - - protected final SqlValidator validator; - protected final RexBuilder rexBuilder; - protected final Prepare.CatalogReader catalogReader; - protected final RelOptCluster cluster; - private DefaultValueFactory defaultValueFactory; - private SubqueryConverter subqueryConverter; - protected final List leaves = new ArrayList(); - private final List dynamicParamSqlNodes = - new ArrayList(); - private final SqlOperatorTable opTab; - private boolean shouldConvertTableAccess; - protected final RelDataTypeFactory typeFactory; - private final SqlNodeToRexConverter exprConverter; - private boolean decorrelationEnabled; - private boolean trimUnusedFields; - private boolean shouldCreateValuesRel; - private boolean isExplain; - private int nDynamicParamsInExplain; - - /** - * Fields used in name resolution for correlated subqueries. - */ - private final Map mapCorrelToDeferred = - new HashMap(); - private int nextCorrel = 0; - - private static final String CORREL_PREFIX = "$cor"; - - /** - * Stack of names of datasets requested by the - * TABLE(SAMPLE(<datasetName>, <query>)) construct. - */ - private final Stack datasetStack = new Stack(); - - /** - * Mapping of non-correlated subqueries that have been converted to their - * equivalent constants. Used to avoid re-evaluating the subquery if it's - * already been evaluated. - */ - private final Map mapConvertedNonCorrSubqs = - new HashMap(); - - public final RelOptTable.ViewExpander viewExpander; - - //~ Constructors ----------------------------------------------------------- - /** - * Creates a converter. - * - * @param viewExpander Preparing statement - * @param validator Validator - * @param catalogReader Schema - * @param planner Planner - * @param rexBuilder Rex builder - * @param convertletTable Expression converter - */ - public SqlToRelConverter( - RelOptTable.ViewExpander viewExpander, - SqlValidator validator, - Prepare.CatalogReader catalogReader, - RelOptPlanner planner, - RexBuilder rexBuilder, - SqlRexConvertletTable convertletTable) { - this.viewExpander = viewExpander; - this.opTab = - (validator - == null) ? SqlStdOperatorTable.instance() - : validator.getOperatorTable(); - this.validator = validator; - this.catalogReader = catalogReader; - this.defaultValueFactory = new NullDefaultValueFactory(); - this.subqueryConverter = new NoOpSubqueryConverter(); - this.rexBuilder = rexBuilder; - this.typeFactory = rexBuilder.getTypeFactory(); - RelOptQuery query = new RelOptQuery(planner); - this.cluster = query.createCluster(typeFactory, rexBuilder); - this.shouldConvertTableAccess = true; - this.exprConverter = - new SqlNodeToRexConverterImpl(convertletTable); - decorrelationEnabled = true; - trimUnusedFields = false; - shouldCreateValuesRel = true; - isExplain = false; - nDynamicParamsInExplain = 0; - } - - //~ Methods ---------------------------------------------------------------- - - /** - * @return the RelOptCluster in use. - */ - public RelOptCluster getCluster() { - return cluster; - } - - /** - * Returns the row-expression builder. - */ - public RexBuilder getRexBuilder() { - return rexBuilder; - } - - /** - * Returns the number of dynamic parameters encountered during translation; - * this must only be called after {@link #convertQuery}. - * - * @return number of dynamic parameters - */ - public int getDynamicParamCount() { - return dynamicParamSqlNodes.size(); - } - - /** - * Returns the type inferred for a dynamic parameter. - * - * @param index 0-based index of dynamic parameter - * @return inferred type, never null - */ - public RelDataType getDynamicParamType(int index) { - SqlNode sqlNode = dynamicParamSqlNodes.get(index); - if (sqlNode == null) { - throw Util.needToImplement("dynamic param type inference"); - } - return validator.getValidatedNodeType(sqlNode); - } - - /** - * Returns the current count of the number of dynamic parameters in an - * EXPLAIN PLAN statement. - * - * @param increment if true, increment the count - * @return the current count before the optional increment - */ - public int getDynamicParamCountInExplain(boolean increment) { - int retVal = nDynamicParamsInExplain; - if (increment) { - ++nDynamicParamsInExplain; - } - return retVal; - } - - /** - * @return mapping of non-correlated subqueries that have been converted to - * the constants that they evaluate to - */ - public Map getMapConvertedNonCorrSubqs() { - return mapConvertedNonCorrSubqs; - } - - /** - * Adds to the current map of non-correlated converted subqueries the - * elements from another map that contains non-correlated subqueries that - * have been converted by another SqlToRelConverter. - * - * @param alreadyConvertedNonCorrSubqs the other map - */ - public void addConvertedNonCorrSubqs( - Map alreadyConvertedNonCorrSubqs) { - mapConvertedNonCorrSubqs.putAll(alreadyConvertedNonCorrSubqs); - } - - /** - * Set a new DefaultValueFactory. To have any effect, this must be called - * before any convert method. - * - * @param factory new DefaultValueFactory - */ - public void setDefaultValueFactory(DefaultValueFactory factory) { - defaultValueFactory = factory; - } - - /** - * Sets a new SubqueryConverter. To have any effect, this must be called - * before any convert method. - * - * @param converter new SubqueryConverter - */ - public void setSubqueryConverter(SubqueryConverter converter) { - subqueryConverter = converter; - } - - /** - * Indicates that the current statement is part of an EXPLAIN PLAN statement - * - * @param nDynamicParams number of dynamic parameters in the statement - */ - public void setIsExplain(int nDynamicParams) { - isExplain = true; - nDynamicParamsInExplain = nDynamicParams; - } - - /** - * Controls whether table access references are converted to physical rels - * immediately. The optimizer doesn't like leaf rels to have - * {@link Convention#NONE}. However, if we are doing further conversion - * passes (e.g. {@link RelStructuredTypeFlattener}), then we may need to - * defer conversion. To have any effect, this must be called before any - * convert method. - * - * @param enabled true for immediate conversion (the default); false to - * generate logical TableAccessRel instances - */ - public void enableTableAccessConversion(boolean enabled) { - shouldConvertTableAccess = enabled; - } - - /** - * Controls whether instances of {@link ValuesRel} are generated. These may - * not be supported by all physical implementations. To have any effect, - * this must be called before any convert method. - * - * @param enabled true to allow ValuesRel to be generated (the default); - * false to force substitution of ProjectRel+OneRowRel instead - */ - public void enableValuesRelCreation(boolean enabled) { - shouldCreateValuesRel = enabled; - } - - private void checkConvertedType(SqlNode query, RelNode result) { - if (!query.isA(SqlKind.DML)) { - // Verify that conversion from SQL to relational algebra did - // not perturb any type information. (We can't do this if the - // SQL statement is something like an INSERT which has no - // validator type information associated with its result, - // hence the namespace check above.) - RelDataType convertedRowType = result.getRowType(); - if (!checkConvertedRowType(query, convertedRowType)) { - RelDataType validatedRowType = - validator.getValidatedNodeType(query); - validatedRowType = uniquifyFields(validatedRowType); - throw Util.newInternal( - "Conversion to relational algebra failed to preserve " - + "datatypes:\n" - + "validated type:\n" - + validatedRowType.getFullTypeString() - + "\nconverted type:\n" - + convertedRowType.getFullTypeString() - + "\nrel:\n" - + RelOptUtil.toString(result)); - } - } - } - - public RelNode flattenTypes( - RelNode rootRel, - boolean restructure) { - RelStructuredTypeFlattener typeFlattener = - new RelStructuredTypeFlattener(rexBuilder, createToRelContext()); - return typeFlattener.rewrite(rootRel, restructure); - } - - /** - * If subquery is correlated and decorrelation is enabled, performs - * decorrelation. - * - * @param query Query - * @param rootRel Root relational expression - * @return New root relational expression after decorrelation - */ - public RelNode decorrelate(SqlNode query, RelNode rootRel) { - if (!enableDecorrelation()) { - return rootRel; - } - final RelNode result = decorrelateQuery(rootRel); - if (result != rootRel) { - checkConvertedType(query, result); - } - return result; - } - - /** - * Walks over a tree of relational expressions, replacing each - * {@link RelNode} with a 'slimmed down' relational expression that projects - * only the fields required by its consumer. - * - *

This may make things easier for the optimizer, by removing crud that - * would expand the search space, but is difficult for the optimizer itself - * to do it, because optimizer rules must preserve the number and type of - * fields. Hence, this transform that operates on the entire tree, similar - * to the {@link RelStructuredTypeFlattener type-flattening transform}. - * - *

Currently this functionality is disabled in farrago/luciddb; the - * default implementation of this method does nothing. - * - * @param rootRel Relational expression that is at the root of the tree - * @return Trimmed relational expression - */ - public RelNode trimUnusedFields(RelNode rootRel) { - // Trim fields that are not used by their consumer. - if (isTrimUnusedFields()) { - final RelFieldTrimmer trimmer = newFieldTrimmer(); - rootRel = trimmer.trim(rootRel); - boolean dumpPlan = SQL2REL_LOGGER.isLoggable(Level.FINE); - if (dumpPlan) { - SQL2REL_LOGGER.fine( - RelOptUtil.dumpPlan( - "Plan after trimming unused fields", - rootRel, - false, - SqlExplainLevel.EXPPLAN_ATTRIBUTES)); - } - } - return rootRel; - } - - /** - * Creates a RelFieldTrimmer. - * - * @return Field trimmer - */ - protected RelFieldTrimmer newFieldTrimmer() { - return new RelFieldTrimmer(validator); - } - - /** - * Converts an unvalidated query's parse tree into a relational expression. - * - * @param query Query to convert - * @param needsValidation Whether to validate the query before converting; - * false if the query has already been - * validated. - * @param top Whether the query is top-level, say if its result - * will become a JDBC result set; false if - * the query will be part of a view. - */ - public RelNode convertQuery( - SqlNode query, - final boolean needsValidation, - final boolean top) { - if (needsValidation) { - query = validator.validate(query); - } - - RelNode result = convertQueryRecursive(query, top, null); - checkConvertedType(query, result); - - boolean dumpPlan = SQL2REL_LOGGER.isLoggable(Level.FINE); - if (dumpPlan) { - SQL2REL_LOGGER.fine( - RelOptUtil.dumpPlan( - "Plan after converting SqlNode to RelNode", - result, - false, - SqlExplainLevel.EXPPLAN_ATTRIBUTES)); - } - - return result; - } - - protected boolean checkConvertedRowType( - SqlNode query, - RelDataType convertedRowType) { - RelDataType validatedRowType = validator.getValidatedNodeType(query); - validatedRowType = uniquifyFields(validatedRowType); - - return RelOptUtil.equal( - "validated row type", - validatedRowType, - "converted row type", - convertedRowType, - false); - } - - protected RelDataType uniquifyFields(RelDataType rowType) { - return validator.getTypeFactory().createStructType( - RelOptUtil.getFieldTypeList(rowType), - SqlValidatorUtil.uniquify(rowType.getFieldNames())); - } - - /** - * Converts a SELECT statement's parse tree into a relational expression. - */ - public RelNode convertSelect(SqlSelect select) { - final SqlValidatorScope selectScope = validator.getWhereScope(select); - final Blackboard bb = createBlackboard(selectScope, null); - convertSelectImpl(bb, select); - return bb.root; - } - - /** - * Factory method for creating translation workspace. - */ - protected Blackboard createBlackboard( - SqlValidatorScope scope, - Map nameToNodeMap) { - return new Blackboard(scope, nameToNodeMap); - } - - /** - * Implementation of {@link #convertSelect(SqlSelect)}; derived class may - * override. - */ - protected void convertSelectImpl( - final Blackboard bb, - SqlSelect select) { - convertFrom( - bb, - select.getFrom()); - convertWhere( - bb, - select.getWhere()); - - List orderExprList = new ArrayList(); - List collationList = - new ArrayList(); - gatherOrderExprs( - bb, - select, - select.getOrderList(), - orderExprList, - collationList); - final RelCollation collation = - cluster.traitSetOf().canonize(RelCollationImpl.of(collationList)); - - if (validator.isAggregate(select)) { - convertAgg( - bb, - select, - orderExprList); - } else { - convertSelectList( - bb, - select, - orderExprList); - } - - if (select.isDistinct()) { - distinctify(bb, true); - } - convertOrder( - select, bb, collation, orderExprList, select.getOffset(), - select.getFetch()); - bb.setRoot(bb.root, true); - } - - /** - * Having translated 'SELECT ... FROM ... [GROUP BY ...] [HAVING ...]', adds - * a relational expression to make the results unique. - * - *

If the SELECT clause contains duplicate expressions, adds {@link - * ProjectRel}s so that we are grouping on the minimal set of keys. The - * performance gain isn't huge, but it is difficult to detect these - * duplicate expressions later. - * - * @param bb Blackboard - * @param checkForDupExprs Check for duplicate expressions - */ - private void distinctify( - Blackboard bb, - boolean checkForDupExprs) { - // Look for duplicate expressions in the project. - // Say we have 'select x, y, x, z'. - // Then dups will be {[2, 0]} - // and oldToNew will be {[0, 0], [1, 1], [2, 0], [3, 2]} - RelNode rel = bb.root; - if (checkForDupExprs && (rel instanceof ProjectRel)) { - ProjectRel project = (ProjectRel) rel; - final List projectExprs = project.getProjects(); - List origins = new ArrayList(); - int dupCount = 0; - for (int i = 0; i < projectExprs.size(); i++) { - int x = findExpr(projectExprs.get(i), projectExprs, i); - if (x >= 0) { - origins.add(x); - ++dupCount; - } else { - origins.add(i); - } - } - if (dupCount == 0) { - distinctify(bb, false); - return; - } - - final Map squished = Maps.newHashMap(); - final List fields = rel.getRowType().getFieldList(); - final List> newProjects = Lists.newArrayList(); - for (int i = 0; i < fields.size(); i++) { - if (origins.get(i) == i) { - squished.put(i, newProjects.size()); - newProjects.add(RexInputRef.of2(i, fields)); - } - } - rel = - new ProjectRel( - cluster, - rel, - Pair.left(newProjects), - Pair.right(newProjects), - ProjectRel.Flags.BOXED); - - bb.root = rel; - distinctify(bb, false); - rel = bb.root; - - // Create the expressions to reverse the mapping. - // Project($0, $1, $0, $2). - final List> undoProjects = Lists.newArrayList(); - for (int i = 0; i < fields.size(); i++) { - final int origin = origins.get(i); - RelDataTypeField field = fields.get(i); - undoProjects.add( - Pair.of( - (RexNode) new RexInputRef( - squished.get(origin), field.getType()), - field.getName())); - } - - rel = - new ProjectRel( - cluster, - rel, - Pair.left(undoProjects), - Pair.right(undoProjects), - ProjectRel.Flags.BOXED); - - bb.setRoot( - rel, - false); - - return; - } - - // Usual case: all of the expressions in the SELECT clause are - // different. - rel = - createAggregate( - bb, - BitSets.range(rel.getRowType().getFieldCount()), - ImmutableList.of()); - - bb.setRoot( - rel, - false); - } - - private int findExpr(RexNode seek, List exprs, int count) { - for (int i = 0; i < count; i++) { - RexNode expr = exprs.get(i); - if (expr.toString().equals(seek.toString())) { - return i; - } - } - return -1; - } - - /** - * Converts a query's ORDER BY clause, if any. - * - * @param select Query - * @param bb Blackboard - * @param collation Collation list - * @param orderExprList Method populates this list with orderBy expressions - * not present in selectList - * @param offset Expression for number of rows to discard before - * returning first row - * @param fetch Expression for number of rows to fetch - */ - protected void convertOrder( - SqlSelect select, - Blackboard bb, - RelCollation collation, - List orderExprList, - SqlNode offset, - SqlNode fetch) { - if (select.getOrderList() == null) { - assert collation.getFieldCollations().isEmpty(); - if (offset == null && fetch == null) { - return; - } - } - - // Create a sorter using the previously constructed collations. - bb.setRoot( - new SortRel( - cluster, - cluster.traitSetOf(Convention.NONE, collation), - bb.root, - collation, - offset == null ? null : convertExpression(offset), - fetch == null ? null : convertExpression(fetch)), - false); - - // If extra expressions were added to the project list for sorting, - // add another project to remove them. - if (orderExprList.size() > 0) { - List exprs = new ArrayList(); - final RelDataType rowType = bb.root.getRowType(); - final int fieldCount = - rowType.getFieldCount() - orderExprList.size(); - for (int i = 0; i < fieldCount; i++) { - exprs.add(rexBuilder.makeInputRef(bb.root, i)); - } - bb.setRoot( - new ProjectRel( - cluster, - cluster.traitSetOf(RelCollationImpl.PRESERVE), - bb.root, - exprs, - cluster.getTypeFactory().createStructType( - rowType.getFieldList().subList(0, fieldCount)), - ProjectRelBase.Flags.BOXED), - false); - } - } - - /** - * Returns whether a given node contains a {@link SqlInOperator}. - * - * @param node a RexNode tree - */ - private static boolean containsInOperator( - SqlNode node) { - try { - SqlVisitor visitor = - new SqlBasicVisitor() { - public Void visit(SqlCall call) { - if (call.getOperator() instanceof SqlInOperator) { - throw new Util.FoundOne(call); - } - return super.visit(call); - } - }; - node.accept(visitor); - return false; - } catch (Util.FoundOne e) { - Util.swallow(e, null); - return true; - } - } - - /** - * Push down all the NOT logical operators into any IN/NOT IN operators. - * - * @param sqlNode the root node from which to look for NOT operators - * @return the transformed SqlNode representation with NOT pushed down. - */ - private static SqlNode pushDownNotForIn(SqlNode sqlNode) { - if ((sqlNode instanceof SqlCall) && containsInOperator(sqlNode)) { - SqlCall sqlCall = (SqlCall) sqlNode; - if ((sqlCall.getOperator() == SqlStdOperatorTable.AND) - || (sqlCall.getOperator() == SqlStdOperatorTable.OR)) { - SqlNode[] sqlOperands = ((SqlBasicCall) sqlCall).operands; - for (int i = 0; i < sqlOperands.length; i++) { - sqlOperands[i] = pushDownNotForIn(sqlOperands[i]); - } - return sqlNode; - } else if (sqlCall.getOperator() == SqlStdOperatorTable.NOT) { - SqlNode childNode = sqlCall.operand(0); - assert childNode instanceof SqlCall; - SqlBasicCall childSqlCall = (SqlBasicCall) childNode; - if (childSqlCall.getOperator() == SqlStdOperatorTable.AND) { - SqlNode[] andOperands = childSqlCall.getOperands(); - SqlNode[] orOperands = new SqlNode[andOperands.length]; - for (int i = 0; i < orOperands.length; i++) { - orOperands[i] = - SqlStdOperatorTable.NOT.createCall( - SqlParserPos.ZERO, - andOperands[i]); - } - for (int i = 0; i < orOperands.length; i++) { - orOperands[i] = pushDownNotForIn(orOperands[i]); - } - return SqlStdOperatorTable.OR.createCall(SqlParserPos.ZERO, - orOperands[0], orOperands[1]); - } else if (childSqlCall.getOperator() == SqlStdOperatorTable.OR) { - SqlNode[] orOperands = childSqlCall.getOperands(); - SqlNode[] andOperands = new SqlNode[orOperands.length]; - for (int i = 0; i < andOperands.length; i++) { - andOperands[i] = - SqlStdOperatorTable.NOT.createCall( - SqlParserPos.ZERO, - orOperands[i]); - } - for (int i = 0; i < andOperands.length; i++) { - andOperands[i] = pushDownNotForIn(andOperands[i]); - } - return SqlStdOperatorTable.AND.createCall(SqlParserPos.ZERO, - andOperands[0], andOperands[1]); - } else if (childSqlCall.getOperator() == SqlStdOperatorTable.NOT) { - SqlNode[] notOperands = childSqlCall.getOperands(); - assert notOperands.length == 1; - return pushDownNotForIn(notOperands[0]); - } else if (childSqlCall.getOperator() instanceof SqlInOperator) { - SqlNode[] inOperands = childSqlCall.getOperands(); - SqlInOperator inOp = - (SqlInOperator) childSqlCall.getOperator(); - if (inOp.isNotIn()) { - return SqlStdOperatorTable.IN.createCall( - SqlParserPos.ZERO, - inOperands[0], - inOperands[1]); - } else { - return SqlStdOperatorTable.NOT_IN.createCall( - SqlParserPos.ZERO, - inOperands[0], - inOperands[1]); - } - } else { - // childSqlCall is "leaf" node in a logical expression tree - // (only considering AND, OR, NOT) - return sqlNode; - } - } else { - // sqlNode is "leaf" node in a logical expression tree - // (only considering AND, OR, NOT) - return sqlNode; - } - } else { - // tree rooted at sqlNode does not contain inOperator - return sqlNode; - } - } - - /** - * Converts a WHERE clause. - * - * @param bb Blackboard - * @param where WHERE clause, may be null - */ - private void convertWhere( - final Blackboard bb, - final SqlNode where) { - if (where == null) { - return; - } - SqlNode newWhere = pushDownNotForIn(where); - replaceSubqueries(bb, newWhere, RelOptUtil.Logic.UNKNOWN_AS_FALSE); - final RexNode convertedWhere = bb.convertExpression(newWhere); - - // only allocate filter if the condition is not TRUE - if (!convertedWhere.isAlwaysTrue()) { - bb.setRoot( - RelOptUtil.createFilter(bb.root, convertedWhere), - false); - } - } - - private void replaceSubqueries( - final Blackboard bb, - final SqlNode expr, - RelOptUtil.Logic logic) { - findSubqueries(bb, expr, logic, false); - for (SubQuery node : bb.subqueryList) { - substituteSubquery(bb, node); - } - } - - private void substituteSubquery(Blackboard bb, SubQuery subQuery) { - final RexNode expr = subQuery.expr; - if (expr != null) { - // Already done. - return; - } - - final SqlBasicCall call; - final RelNode rel; - final SqlNode query; - final Pair converted; - switch (subQuery.node.getKind()) { - case CURSOR: - convertCursor(bb, subQuery); - return; - - case MULTISET_QUERY_CONSTRUCTOR: - case MULTISET_VALUE_CONSTRUCTOR: - rel = convertMultisets(ImmutableList.of(subQuery.node), bb); - subQuery.expr = bb.register(rel, JoinRelType.INNER); - return; - - case IN: - call = (SqlBasicCall) subQuery.node; - final SqlNode[] operands = call.getOperands(); - - SqlNode leftKeyNode = operands[0]; - query = operands[1]; - - final List leftKeys; - switch (leftKeyNode.getKind()) { - case ROW: - leftKeys = Lists.newArrayList(); - for (SqlNode sqlExpr : ((SqlBasicCall) leftKeyNode).getOperandList()) { - leftKeys.add(bb.convertExpression(sqlExpr)); - } - break; - default: - leftKeys = ImmutableList.of(bb.convertExpression(leftKeyNode)); - } - - final boolean isNotIn = ((SqlInOperator) call.getOperator()).isNotIn(); - if (query instanceof SqlNodeList) { - SqlNodeList valueList = (SqlNodeList) query; - if (!containsNullLiteral(valueList) - && valueList.size() < getInSubqueryThreshold()) { - // We're under the threshold, so convert to OR. - subQuery.expr = - convertInToOr( - bb, - leftKeys, - valueList, - isNotIn); - return; - } - - // Otherwise, let convertExists translate - // values list into an inline table for the - // reference to Q below. - } - - // Project out the search columns from the left side - - // Q1: - // "select from emp where emp.deptno in (select col1 from T)" - // - // is converted to - // - // "select from - // emp inner join (select distinct col1 from T)) q - // on emp.deptno = q.col1 - // - // Q2: - // "select from emp where emp.deptno not in (Q)" - // - // is converted to - // - // "select from - // emp left outer join (select distinct col1, TRUE from T) q - // on emp.deptno = q.col1 - // where emp.deptno <> null - // and q.indicator <> TRUE" - // - final boolean outerJoin = bb.subqueryNeedsOuterJoin - || isNotIn - || subQuery.logic == RelOptUtil.Logic.TRUE_FALSE_UNKNOWN; - converted = - convertExists(query, RelOptUtil.SubqueryType.IN, subQuery.logic, - outerJoin); - if (converted.right) { - // Generate - // emp CROSS JOIN (SELECT COUNT(*) AS c, - // COUNT(deptno) AS ck FROM dept) - final RelDataType longType = - typeFactory.createSqlType(SqlTypeName.BIGINT); - final RelNode seek = converted.left.getInput(0); // fragile - final int keyCount = leftKeys.size(); - final List args = ImmutableIntList.range(0, keyCount); - AggregateRel aggregate = - new AggregateRel(cluster, seek, BitSets.of(), - ImmutableList.of( - new AggregateCall(SqlStdOperatorTable.COUNT, false, - ImmutableList.of(), longType, null), - new AggregateCall(SqlStdOperatorTable.COUNT, false, - args, longType, null))); - JoinRel join = - new JoinRel(cluster, bb.root, aggregate, - rexBuilder.makeLiteral(true), JoinRelType.INNER, - ImmutableSet.of()); - bb.setRoot(join, false); - } - RexNode rex = - bb.register(converted.left, - outerJoin ? JoinRelType.LEFT : JoinRelType.INNER, leftKeys); - - subQuery.expr = translateIn(subQuery, bb.root, rex); - if (isNotIn) { - subQuery.expr = - rexBuilder.makeCall(SqlStdOperatorTable.NOT, subQuery.expr); - } - return; - - case EXISTS: - // "select from emp where exists (select a from T)" - // - // is converted to the following if the subquery is correlated: - // - // "select from emp left outer join (select AGG_TRUE() as indicator - // from T group by corr_var) q where q.indicator is true" - // - // If there is no correlation, the expression is replaced with a - // boolean indicating whether the subquery returned 0 or >= 1 row. - call = (SqlBasicCall) subQuery.node; - query = call.getOperands()[0]; - converted = convertExists(query, RelOptUtil.SubqueryType.EXISTS, - subQuery.logic, true); - assert !converted.right; - if (convertNonCorrelatedSubQuery(subQuery, bb, converted.left, true)) { - return; - } - subQuery.expr = bb.register(converted.left, JoinRelType.LEFT); - return; - - case SCALAR_QUERY: - // Convert the subquery. If it's non-correlated, convert it - // to a constant expression. - call = (SqlBasicCall) subQuery.node; - query = call.getOperands()[0]; - converted = convertExists(query, RelOptUtil.SubqueryType.SCALAR, - subQuery.logic, true); - assert !converted.right; - if (convertNonCorrelatedSubQuery(subQuery, bb, converted.left, false)) { - return; - } - rel = convertToSingleValueSubq(query, converted.left); - subQuery.expr = bb.register(rel, JoinRelType.LEFT); - return; - - case SELECT: - // This is used when converting multiset queries: - // - // select * from unnest(select multiset[deptno] from emps); - // - converted = convertExists(subQuery.node, RelOptUtil.SubqueryType.SCALAR, - subQuery.logic, true); - assert !converted.right; - subQuery.expr = bb.register(converted.left, JoinRelType.LEFT); - return; - - default: - throw Util.newInternal("unexpected kind of subquery :" + subQuery.node); - } - } - - private RexNode translateIn(SubQuery subQuery, RelNode root, - final RexNode rex) { - switch (subQuery.logic) { - case TRUE: - return rexBuilder.makeLiteral(true); - - case UNKNOWN_AS_FALSE: - assert rex instanceof RexRangeRef; - final int fieldCount = rex.getType().getFieldCount(); - RexNode rexNode = rexBuilder.makeFieldAccess(rex, fieldCount - 1); - rexNode = rexBuilder.makeCall(SqlStdOperatorTable.IS_TRUE, rexNode); - - // Then append the IS NOT NULL(leftKeysForIn). - // - // RexRangeRef contains the following fields: - // leftKeysForIn, - // rightKeysForIn (the original subquery select list), - // nullIndicator - // - // The first two lists contain the same number of fields. - final int k = (fieldCount - 1) / 2; - for (int i = 0; i < k; i++) { - rexNode = - rexBuilder.makeCall( - SqlStdOperatorTable.AND, - rexNode, - rexBuilder.makeCall( - SqlStdOperatorTable.IS_NOT_NULL, - rexBuilder.makeFieldAccess(rex, i))); - } - return rexNode; - - case TRUE_FALSE_UNKNOWN: - case UNKNOWN_AS_TRUE: - // select e.deptno, - // case - // when ct.c = 0 then false - // when dt.i is not null then true - // when e.deptno is null then null - // when ct.ck < ct.c then null - // else false - // end - // from e - // cross join (select count(*) as c, count(deptno) as ck from v) as ct - // left join (select distinct deptno, true as i from v) as dt - // on e.deptno = dt.deptno - final JoinRelBase join = (JoinRelBase) root; - final ProjectRelBase left = (ProjectRelBase) join.getLeft(); - final RelNode leftLeft = ((JoinRelBase) left.getInput(0)).getLeft(); - final int leftLeftCount = leftLeft.getRowType().getFieldCount(); - final RelDataType nullableBooleanType = - typeFactory.createTypeWithNullability( - typeFactory.createSqlType(SqlTypeName.BOOLEAN), true); - final RelDataType longType = - typeFactory.createSqlType(SqlTypeName.BIGINT); - final RexNode cRef = rexBuilder.makeInputRef(root, leftLeftCount); - final RexNode ckRef = rexBuilder.makeInputRef(root, leftLeftCount + 1); - final RexNode iRef = - rexBuilder.makeInputRef(root, root.getRowType().getFieldCount() - 1); - - final RexLiteral zero = - rexBuilder.makeExactLiteral(BigDecimal.ZERO, longType); - final RexLiteral trueLiteral = rexBuilder.makeLiteral(true); - final RexLiteral falseLiteral = rexBuilder.makeLiteral(false); - final RexNode unknownLiteral = - rexBuilder.makeNullLiteral(SqlTypeName.BOOLEAN); - - final ImmutableList.Builder args = ImmutableList.builder(); - args.add(rexBuilder.makeCall(SqlStdOperatorTable.EQUALS, cRef, zero), - falseLiteral, - rexBuilder.makeCall(SqlStdOperatorTable.IS_NOT_NULL, iRef), - trueLiteral); - final JoinInfo joinInfo = join.analyzeCondition(); - for (int leftKey : joinInfo.leftKeys) { - final RexNode kRef = rexBuilder.makeInputRef(root, leftKey); - args.add(rexBuilder.makeCall(SqlStdOperatorTable.IS_NULL, kRef), - unknownLiteral); - } - args.add(rexBuilder.makeCall(SqlStdOperatorTable.LESS_THAN, ckRef, cRef), - unknownLiteral, - falseLiteral); - - return rexBuilder.makeCall( - nullableBooleanType, - SqlStdOperatorTable.CASE, - args.build()); - - default: - throw new AssertionError(subQuery.logic); - } - } - - private static boolean containsNullLiteral(SqlNodeList valueList) { - for (SqlNode node : valueList.getList()) { - if (node instanceof SqlLiteral) { - SqlLiteral lit = (SqlLiteral) node; - if (lit.getValue() == null) { - return true; - } - } - } - return false; - } - - /** - * Determines if a subquery is non-correlated and if so, converts it to a - * constant. - * - * @param subQuery the call that references the subquery - * @param bb blackboard used to convert the subquery - * @param converted RelNode tree corresponding to the subquery - * @param isExists true if the subquery is part of an EXISTS expression - * @return if the subquery can be converted to a constant - */ - private boolean convertNonCorrelatedSubQuery( - SubQuery subQuery, - Blackboard bb, - RelNode converted, - boolean isExists) { - SqlCall call = (SqlBasicCall) subQuery.node; - if (subqueryConverter.canConvertSubquery() - && isSubqNonCorrelated(converted, bb)) { - // First check if the subquery has already been converted - // because it's a nested subquery. If so, don't re-evaluate - // it again. - RexNode constExpr = mapConvertedNonCorrSubqs.get(call); - if (constExpr == null) { - constExpr = - subqueryConverter.convertSubquery( - call, - this, - isExists, - isExplain); - } - if (constExpr != null) { - subQuery.expr = constExpr; - mapConvertedNonCorrSubqs.put(call, constExpr); - return true; - } - } - return false; - } - - /** - * Converts the RelNode tree for a select statement to a select that - * produces a single value. - * - * @param query the query - * @param plan the original RelNode tree corresponding to the statement - * @return the converted RelNode tree - */ - public RelNode convertToSingleValueSubq( - SqlNode query, - RelNode plan) { - // Check whether query is guaranteed to produce a single value. - if (query instanceof SqlSelect) { - SqlSelect select = (SqlSelect) query; - SqlNodeList selectList = select.getSelectList(); - SqlNodeList groupList = select.getGroup(); - - if ((selectList.size() == 1) - && ((groupList == null) || (groupList.size() == 0))) { - SqlNode selectExpr = selectList.get(0); - if (selectExpr instanceof SqlCall) { - SqlCall selectExprCall = (SqlCall) selectExpr; - if (selectExprCall.getOperator() - instanceof SqlAggFunction) { - return plan; - } - } - } - } - - // If not, project SingleValueAgg - return RelOptUtil.createSingleValueAggRel( - cluster, - plan); - } - - /** - * Converts "x IN (1, 2, ...)" to "x=1 OR x=2 OR ...". - * - * @param leftKeys LHS - * @param valuesList RHS - * @param isNotIn is this a NOT IN operator - * @return converted expression - */ - private RexNode convertInToOr( - final Blackboard bb, - final List leftKeys, - SqlNodeList valuesList, - boolean isNotIn) { - List comparisons = new ArrayList(); - for (SqlNode rightVals : valuesList) { - RexNode rexComparison; - if (leftKeys.size() == 1) { - rexComparison = - rexBuilder.makeCall( - SqlStdOperatorTable.EQUALS, - leftKeys.get(0), - bb.convertExpression(rightVals)); - } else { - assert rightVals instanceof SqlCall; - final SqlBasicCall call = (SqlBasicCall) rightVals; - assert (call.getOperator() instanceof SqlRowOperator) - && call.getOperands().length == leftKeys.size(); - rexComparison = - RexUtil.composeConjunction( - rexBuilder, - Iterables.transform( - Pair.zip(leftKeys, call.getOperandList()), - new Function, RexNode>() { - public RexNode apply(Pair pair) { - return rexBuilder.makeCall(SqlStdOperatorTable.EQUALS, - pair.left, bb.convertExpression(pair.right)); - } - }), - false); - } - comparisons.add(rexComparison); - } - - RexNode result = - RexUtil.composeDisjunction(rexBuilder, comparisons, true); - assert result != null; - - if (isNotIn) { - result = - rexBuilder.makeCall( - SqlStdOperatorTable.NOT, - result); - } - - return result; - } - - /** - * Gets the list size threshold under which {@link #convertInToOr} is used. - * Lists of this size or greater will instead be converted to use a join - * against an inline table ({@link ValuesRel}) rather than a predicate. A - * threshold of 0 forces usage of an inline table in all cases; a threshold - * of Integer.MAX_VALUE forces usage of OR in all cases - * - * @return threshold, default 20 - */ - protected int getInSubqueryThreshold() { - // OVERRIDE POINT - return Integer.MAX_VALUE; // was 20 - } - - /** - * Converts an EXISTS or IN predicate into a join. For EXISTS, the subquery - * produces an indicator variable, and the result is a relational expression - * which outer joins that indicator to the original query. After performing - * the outer join, the condition will be TRUE if the EXISTS condition holds, - * NULL otherwise. - * - * @param seek A query, for example 'select * from emp' or - * 'values (1,2,3)' or '('Foo', 34)'. - * @param subqueryType Whether sub-query is IN, EXISTS or scalar - * @param logic Whether the answer needs to be in full 3-valued logic (TRUE, - * FALSE, UNKNOWN) will be required, or whether we can accept an - * approximation (say representing UNKNOWN as FALSE) - * @param needsOuterJoin Whether an outer join is needed - * @return join expression - * @pre extraExpr == null || extraName != null - */ - private Pair convertExists( - SqlNode seek, - RelOptUtil.SubqueryType subqueryType, - RelOptUtil.Logic logic, - boolean needsOuterJoin) { - final SqlValidatorScope seekScope = - (seek instanceof SqlSelect) - ? validator.getSelectScope((SqlSelect) seek) - : null; - final Blackboard seekBb = createBlackboard(seekScope, null); - RelNode seekRel = convertQueryOrInList(seekBb, seek); - - return RelOptUtil.createExistsPlan(seekRel, subqueryType, logic, - needsOuterJoin); - } - - private RelNode convertQueryOrInList( - Blackboard bb, - SqlNode seek) { - // NOTE: Once we start accepting single-row queries as row constructors, - // there will be an ambiguity here for a case like X IN ((SELECT Y FROM - // Z)). The SQL standard resolves the ambiguity by saying that a lone - // select should be interpreted as a table expression, not a row - // expression. The semantic difference is that a table expression can - // return multiple rows. - if (seek instanceof SqlNodeList) { - return convertRowValues( - bb, - seek, - ((SqlNodeList) seek).getList(), - false, - null); - } else { - return convertQueryRecursive(seek, false, null); - } - } - - private RelNode convertRowValues( - Blackboard bb, - SqlNode rowList, - Collection rows, - boolean allowLiteralsOnly, - RelDataType targetRowType) { - // NOTE jvs 30-Apr-2006: We combine all rows consisting entirely of - // literals into a single ValuesRel; this gives the optimizer a smaller - // input tree. For everything else (computed expressions, row - // subqueries), we union each row in as a projection on top of a - // OneRowRel. - - final List> tupleList = - new ArrayList>(); - final RelDataType rowType; - if (targetRowType != null) { - rowType = targetRowType; - } else { - rowType = - SqlTypeUtil.promoteToRowType( - typeFactory, - validator.getValidatedNodeType(rowList), - null); - } - - List unionInputs = new ArrayList(); - for (SqlNode node : rows) { - SqlBasicCall call; - if (isRowConstructor(node)) { - call = (SqlBasicCall) node; - List tuple = new ArrayList(); - for (SqlNode operand : call.operands) { - RexLiteral rexLiteral = - convertLiteralInValuesList( - operand, - bb, - rowType, - tuple.size()); - if ((rexLiteral == null) && allowLiteralsOnly) { - return null; - } - if ((rexLiteral == null) || !shouldCreateValuesRel) { - // fallback to convertRowConstructor - tuple = null; - break; - } - tuple.add(rexLiteral); - } - if (tuple != null) { - tupleList.add(tuple); - continue; - } - } else { - RexLiteral rexLiteral = - convertLiteralInValuesList( - node, - bb, - rowType, - 0); - if ((rexLiteral != null) && shouldCreateValuesRel) { - tupleList.add( - Collections.singletonList(rexLiteral)); - continue; - } else { - if ((rexLiteral == null) && allowLiteralsOnly) { - return null; - } - } - - // convert "1" to "row(1)" - call = - (SqlBasicCall) SqlStdOperatorTable.ROW.createCall( - SqlParserPos.ZERO, - node); - } - unionInputs.add(convertRowConstructor(bb, call)); - } - ValuesRel valuesRel = - new ValuesRel( - cluster, - rowType, - tupleList); - RelNode resultRel; - if (unionInputs.isEmpty()) { - resultRel = valuesRel; - } else { - if (!tupleList.isEmpty()) { - unionInputs.add(valuesRel); - } - UnionRel unionRel = - new UnionRel( - cluster, - unionInputs, - true); - resultRel = unionRel; - } - leaves.add(resultRel); - return resultRel; - } - - private RexLiteral convertLiteralInValuesList( - SqlNode sqlNode, - Blackboard bb, - RelDataType rowType, - int iField) { - if (!(sqlNode instanceof SqlLiteral)) { - return null; - } - RelDataTypeField field = rowType.getFieldList().get(iField); - RelDataType type = field.getType(); - if (type.isStruct()) { - // null literals for weird stuff like UDT's need - // special handling during type flattening, so - // don't use ValuesRel for those - return null; - } - - RexNode literalExpr = - exprConverter.convertLiteral( - bb, - (SqlLiteral) sqlNode); - - if (!(literalExpr instanceof RexLiteral)) { - assert literalExpr.isA(SqlKind.CAST); - RexNode child = ((RexCall) literalExpr).getOperands().get(0); - assert RexLiteral.isNullLiteral(child); - - // NOTE jvs 22-Nov-2006: we preserve type info - // in ValuesRel digest, so it's OK to lose it here - return (RexLiteral) child; - } - - RexLiteral literal = (RexLiteral) literalExpr; - - Comparable value = literal.getValue(); - - if (SqlTypeUtil.isExactNumeric(type)) { - BigDecimal roundedValue = - NumberUtil.rescaleBigDecimal( - (BigDecimal) value, - type.getScale()); - return rexBuilder.makeExactLiteral( - roundedValue, - type); - } - - if ((value instanceof NlsString) - && (type.getSqlTypeName() == SqlTypeName.CHAR)) { - // pad fixed character type - NlsString unpadded = (NlsString) value; - return rexBuilder.makeCharLiteral( - new NlsString( - Util.rpad(unpadded.getValue(), type.getPrecision()), - unpadded.getCharsetName(), - unpadded.getCollation())); - } - return literal; - } - - private boolean isRowConstructor(SqlNode node) { - if (!(node.getKind() == SqlKind.ROW)) { - return false; - } - SqlCall call = (SqlCall) node; - return call.getOperator().getName().equalsIgnoreCase("row"); - } - - /** - * Builds a list of all IN or EXISTS operators - * inside SQL parse tree. Does not traverse inside queries. - * - * @param bb blackboard - * @param node the SQL parse tree - * @param logic Whether the answer needs to be in full 3-valued logic (TRUE, - * FALSE, UNKNOWN) will be required, or whether we can accept - * an approximation (say representing UNKNOWN as FALSE) - * @param registerOnlyScalarSubqueries if set to true and the parse tree - * corresponds to a variation of a select - * node, only register it if it's a scalar - * subquery - */ -private void findSubqueries( - Blackboard bb, - SqlNode node, - RelOptUtil.Logic logic, - boolean registerOnlyScalarSubqueries) { - final SqlKind kind = node.getKind(); - switch (kind) { - case EXISTS: - case SELECT: - case MULTISET_QUERY_CONSTRUCTOR: - case MULTISET_VALUE_CONSTRUCTOR: - case CURSOR: - case SCALAR_QUERY: - if (!registerOnlyScalarSubqueries - || (kind == SqlKind.SCALAR_QUERY)) { - bb.registerSubquery(node, RelOptUtil.Logic.TRUE_FALSE); - } - return; - case IN: - if (((SqlCall) node).getOperator() == SqlStdOperatorTable.NOT_IN) { - logic = logic.negate(); - } - break; - case NOT: - logic = logic.negate(); - break; - } - if (node instanceof SqlCall) { - if (kind == SqlKind.OR - || kind == SqlKind.NOT) { - // It's always correct to outer join subquery with - // containing query; however, when predicates involve Or - // or NOT, outer join might be necessary. - bb.subqueryNeedsOuterJoin = true; - } - for (SqlNode operand : ((SqlCall) node).getOperandList()) { - if (operand != null) { - // In the case of an IN expression, locate scalar - // subqueries so we can convert them to constants - findSubqueries( - bb, - operand, - logic, - kind == SqlKind.IN || registerOnlyScalarSubqueries); - } - } - } else if (node instanceof SqlNodeList) { - for (SqlNode child : (SqlNodeList) node) { - findSubqueries( - bb, - child, - logic, - kind == SqlKind.IN || registerOnlyScalarSubqueries); - } - } - - // Now that we've located any scalar subqueries inside the IN - // expression, register the IN expression itself. We need to - // register the scalar subqueries first so they can be converted - // before the IN expression is converted. - if (kind == SqlKind.IN) { - if (logic == RelOptUtil.Logic.TRUE_FALSE_UNKNOWN - && !validator.getValidatedNodeType(node).isNullable()) { - logic = RelOptUtil.Logic.UNKNOWN_AS_FALSE; - } - // TODO: This conversion is only valid in the WHERE clause - if (logic == RelOptUtil.Logic.UNKNOWN_AS_FALSE - && !bb.subqueryNeedsOuterJoin) { - logic = RelOptUtil.Logic.TRUE; - } - bb.registerSubquery(node, logic); - } - } - - /** - * Converts an expression from {@link SqlNode} to {@link RexNode} format. - * - * @param node Expression to translate - * @return Converted expression - */ - public RexNode convertExpression( - SqlNode node) { - Map nameToTypeMap = Collections.emptyMap(); - Blackboard bb = - createBlackboard( - new ParameterScope((SqlValidatorImpl) validator, nameToTypeMap), - null); - return bb.convertExpression(node); - } - - /** - * Converts an expression from {@link SqlNode} to {@link RexNode} format, - * mapping identifier references to predefined expressions. - * - * @param node Expression to translate - * @param nameToNodeMap map from String to {@link RexNode}; when an - * {@link SqlIdentifier} is encountered, it is used as a - * key and translated to the corresponding value from - * this map - * @return Converted expression - */ - public RexNode convertExpression( - SqlNode node, - Map nameToNodeMap) { - final Map nameToTypeMap = - new HashMap(); - for (Map.Entry entry : nameToNodeMap.entrySet()) { - nameToTypeMap.put(entry.getKey(), entry.getValue().getType()); - } - Blackboard bb = - createBlackboard( - new ParameterScope((SqlValidatorImpl) validator, nameToTypeMap), - nameToNodeMap); - return bb.convertExpression(node); - } - - /** - * Converts a non-standard expression. - * - *

This method is an extension-point that derived classes can override. If - * this method returns a null result, the normal expression translation - * process will proceed. The default implementation always returns null. - * - * @param node Expression - * @param bb Blackboard - * @return null to proceed with the usual expression translation process - */ - protected RexNode convertExtendedExpression( - SqlNode node, - Blackboard bb) { - return null; - } - - private RexNode convertOver(Blackboard bb, SqlNode node) { - SqlCall call = (SqlCall) node; - SqlCall aggCall = call.operand(0); - SqlNode windowOrRef = call.operand(1); - final SqlWindow window = - validator.resolveWindow(windowOrRef, bb.scope, true); - final SqlNodeList partitionList = window.getPartitionList(); - final ImmutableList.Builder partitionKeys = - ImmutableList.builder(); - for (SqlNode partition : partitionList) { - partitionKeys.add(bb.convertExpression(partition)); - } - RexNode lowerBound = bb.convertExpression(window.getLowerBound()); - RexNode upperBound = bb.convertExpression(window.getUpperBound()); - SqlNodeList orderList = window.getOrderList(); - if ((orderList.size() == 0) && !window.isRows()) { - // A logical range requires an ORDER BY clause. Use the implicit - // ordering of this relation. There must be one, otherwise it would - // have failed validation. - orderList = bb.scope.getOrderList(); - if (orderList == null) { - throw new AssertionError( - "Relation should have sort key for implicit ORDER BY"); - } - } - final ImmutableList.Builder orderKeys = - ImmutableList.builder(); - final Set flags = EnumSet.noneOf(SqlKind.class); - for (SqlNode order : orderList) { - flags.clear(); - RexNode e = bb.convertSortExpression(order, flags); - orderKeys.add(new RexFieldCollation(e, flags)); - } - try { - Util.permAssert(bb.window == null, "already in window agg mode"); - bb.window = window; - RexNode rexAgg = exprConverter.convertCall(bb, aggCall); - rexAgg = - rexBuilder.ensureType( - validator.getValidatedNodeType(call), rexAgg, false); - - // Walk over the tree and apply 'over' to all agg functions. This is - // necessary because the returned expression is not necessarily a call - // to an agg function. For example, AVG(x) becomes SUM(x) / COUNT(x). - final RexShuttle visitor = - new HistogramShuttle( - partitionKeys.build(), orderKeys.build(), - RexWindowBound.create(window.getLowerBound(), lowerBound), - RexWindowBound.create(window.getUpperBound(), upperBound), - window); - return rexAgg.accept(visitor); - } finally { - bb.window = null; - } - } - - /** - * Converts a FROM clause into a relational expression. - * - * @param bb Scope within which to resolve identifiers - * @param from FROM clause of a query. Examples include: - * - *

    - *
  • a single table ("SALES.EMP"), - *
  • an aliased table ("EMP AS E"), - *
  • a list of tables ("EMP, DEPT"), - *
  • an ANSI Join expression ("EMP JOIN DEPT ON EMP.DEPTNO = - * DEPT.DEPTNO"), - *
  • a VALUES clause ("VALUES ('Fred', 20)"), - *
  • a query ("(SELECT * FROM EMP WHERE GENDER = 'F')"), - *
  • or any combination of the above. - *
- */ - protected void convertFrom( - Blackboard bb, - SqlNode from) { - SqlCall call; - final SqlNode[] operands; - switch (from.getKind()) { - case AS: - operands = ((SqlBasicCall) from).getOperands(); - convertFrom(bb, operands[0]); - return; - - case WITH_ITEM: - convertFrom(bb, ((SqlWithItem) from).query); - return; - - case WITH: - convertFrom(bb, ((SqlWith) from).body); - return; - - case TABLESAMPLE: - operands = ((SqlBasicCall) from).getOperands(); - SqlSampleSpec sampleSpec = SqlLiteral.sampleValue(operands[1]); - if (sampleSpec instanceof SqlSampleSpec.SqlSubstitutionSampleSpec) { - String sampleName = - ((SqlSampleSpec.SqlSubstitutionSampleSpec) sampleSpec) - .getName(); - datasetStack.push(sampleName); - convertFrom(bb, operands[0]); - datasetStack.pop(); - } else if (sampleSpec instanceof SqlSampleSpec.SqlTableSampleSpec) { - SqlSampleSpec.SqlTableSampleSpec tableSampleSpec = - (SqlSampleSpec.SqlTableSampleSpec) sampleSpec; - convertFrom(bb, operands[0]); - RelOptSamplingParameters params = - new RelOptSamplingParameters( - tableSampleSpec.isBernoulli(), - tableSampleSpec.getSamplePercentage(), - tableSampleSpec.isRepeatable(), - tableSampleSpec.getRepeatableSeed()); - bb.setRoot(new SamplingRel(cluster, bb.root, params), false); - } else { - throw Util.newInternal( - "unknown TABLESAMPLE type: " + sampleSpec); - } - return; - - case IDENTIFIER: - final SqlValidatorNamespace fromNamespace = - validator.getNamespace(from).resolve(); - if (fromNamespace.getNode() != null) { - convertFrom(bb, fromNamespace.getNode()); - return; - } - final String datasetName = - datasetStack.isEmpty() ? null : datasetStack.peek(); - boolean[] usedDataset = {false}; - RelOptTable table = - SqlValidatorUtil.getRelOptTable( - fromNamespace, - catalogReader, - datasetName, - usedDataset); - final RelNode tableRel; - if (shouldConvertTableAccess) { - tableRel = toRel(table); - } else { - tableRel = new TableAccessRel(cluster, table); - } - bb.setRoot(tableRel, true); - if (usedDataset[0]) { - bb.setDataset(datasetName); - } - return; - - case JOIN: - final SqlJoin join = (SqlJoin) from; - final Blackboard fromBlackboard = - createBlackboard(validator.getJoinScope(from), null); - SqlNode left = join.getLeft(); - SqlNode right = join.getRight(); - final boolean isNatural = join.isNatural(); - final JoinType joinType = join.getJoinType(); - final Blackboard leftBlackboard = - createBlackboard( - Util.first(validator.getJoinScope(left), - ((DelegatingScope) bb.scope).getParent()), null); - final Blackboard rightBlackboard = - createBlackboard( - Util.first(validator.getJoinScope(right), - ((DelegatingScope) bb.scope).getParent()), null); - convertFrom(leftBlackboard, left); - RelNode leftRel = leftBlackboard.root; - convertFrom(rightBlackboard, right); - RelNode rightRel = rightBlackboard.root; - JoinRelType convertedJoinType = convertJoinType(joinType); - RexNode conditionExp; - if (isNatural) { - final List columnList = - SqlValidatorUtil.deriveNaturalJoinColumnList( - validator.getNamespace(left).getRowType(), - validator.getNamespace(right).getRowType()); - conditionExp = convertUsing(leftRel, rightRel, columnList); - } else { - conditionExp = - convertJoinCondition( - fromBlackboard, - join.getCondition(), - join.getConditionType(), - leftRel, - rightRel); - } - - final RelNode joinRel = - createJoin( - fromBlackboard, - leftRel, - rightRel, - conditionExp, - convertedJoinType); - bb.setRoot(joinRel, false); - return; - - case SELECT: - case INTERSECT: - case EXCEPT: - case UNION: - final RelNode rel = convertQueryRecursive(from, false, null); - bb.setRoot(rel, true); - return; - - case VALUES: - convertValuesImpl(bb, (SqlCall) from, null); - return; - - case UNNEST: - final SqlNode node = ((SqlCall) from).operand(0); - replaceSubqueries(bb, node, RelOptUtil.Logic.TRUE_FALSE_UNKNOWN); - final RelNode childRel = - RelOptUtil.createProject( - (null != bb.root) ? bb.root : new OneRowRel(cluster), - Collections.singletonList(bb.convertExpression(node)), - Collections.singletonList(validator.deriveAlias(node, 0)), - true); - - UncollectRel uncollectRel = - new UncollectRel(cluster, cluster.traitSetOf(Convention.NONE), - childRel); - bb.setRoot(uncollectRel, true); - return; - - case COLLECTION_TABLE: - call = (SqlCall) from; - - // Dig out real call; TABLE() wrapper is just syntactic. - assert call.getOperandList().size() == 1; - call = call.operand(0); - convertCollectionTable(bb, call); - return; - - default: - throw Util.newInternal("not a join operator " + from); - } - } - - protected void convertCollectionTable( - Blackboard bb, - SqlCall call) { - final SqlOperator operator = call.getOperator(); - if (operator == SqlStdOperatorTable.TABLESAMPLE) { - final String sampleName = - SqlLiteral.stringValue(call.operand(0)); - datasetStack.push(sampleName); - SqlCall cursorCall = call.operand(1); - SqlNode query = cursorCall.operand(0); - RelNode converted = convertQuery(query, false, false); - bb.setRoot(converted, false); - datasetStack.pop(); - return; - } - replaceSubqueries(bb, call, RelOptUtil.Logic.TRUE_FALSE_UNKNOWN); - - // Expand table macro if possible. It's more efficient than - // TableFunctionRel. - if (operator instanceof SqlUserDefinedTableMacro) { - final SqlUserDefinedTableMacro udf = - (SqlUserDefinedTableMacro) operator; - final TranslatableTable table = udf.getTable(typeFactory, - call.getOperandList()); - final RelDataType rowType = table.getRowType(typeFactory); - RelOptTable relOptTable = RelOptTableImpl.create(null, rowType, table); - RelNode converted = toRel(relOptTable); - bb.setRoot(converted, true); - return; - } - - Type elementType; - if (operator instanceof SqlUserDefinedTableFunction) { - SqlUserDefinedTableFunction udtf = (SqlUserDefinedTableFunction) operator; - elementType = udtf.getElementType(typeFactory, call.getOperandList()); - } else { - elementType = null; - } - - RexNode rexCall = bb.convertExpression(call); - final List inputs = bb.retrieveCursors(); - Set columnMappings = - getColumnMappings(operator); - TableFunctionRel callRel = - new TableFunctionRel( - cluster, - inputs, - rexCall, - elementType, - validator.getValidatedNodeType(call), - columnMappings); - bb.setRoot(callRel, true); - afterTableFunction(bb, call, callRel); - } - - protected void afterTableFunction( - SqlToRelConverter.Blackboard bb, - SqlCall call, - TableFunctionRel callRel) { - } - - private Set getColumnMappings(SqlOperator op) { - SqlReturnTypeInference rti = op.getReturnTypeInference(); - if (rti == null) { - return null; - } - if (rti instanceof TableFunctionReturnTypeInference) { - TableFunctionReturnTypeInference tfrti = - (TableFunctionReturnTypeInference) rti; - return tfrti.getColumnMappings(); - } else { - return null; - } - } - - protected RelNode createJoin( - Blackboard bb, - RelNode leftRel, - RelNode rightRel, - RexNode joinCond, - JoinRelType joinType) { - assert joinCond != null; - - Set correlatedVariables = RelOptUtil.getVariablesUsed(rightRel); - if (correlatedVariables.size() > 0) { - final List correlations = Lists.newArrayList(); - - for (String correlName : correlatedVariables) { - DeferredLookup lookup = mapCorrelToDeferred.get(correlName); - RexFieldAccess fieldAccess = lookup.getFieldAccess(correlName); - String originalRelName = lookup.getOriginalRelName(); - String originalFieldName = fieldAccess.getField().getName(); - - int[] nsIndexes = {-1}; - final SqlValidatorScope[] ancestorScopes = {null}; - SqlValidatorNamespace foundNs = - lookup.bb.scope.resolve( - originalRelName, - ancestorScopes, - nsIndexes); - - assert foundNs != null; - assert nsIndexes.length == 1; - - int childNamespaceIndex = nsIndexes[0]; - - SqlValidatorScope ancestorScope = ancestorScopes[0]; - boolean correlInCurrentScope = ancestorScope == bb.scope; - - if (correlInCurrentScope) { - int namespaceOffset = 0; - if (childNamespaceIndex > 0) { - // If not the first child, need to figure out the width - // of output types from all the preceding namespaces - assert ancestorScope instanceof ListScope; - List children = - ((ListScope) ancestorScope).getChildren(); - - for (int i = 0; i < childNamespaceIndex; i++) { - SqlValidatorNamespace child = children.get(i); - namespaceOffset += - child.getRowType().getFieldCount(); - } - } - - RelDataTypeField field = - catalogReader.field(foundNs.getRowType(), originalFieldName); - int pos = namespaceOffset + field.getIndex(); - - assert field.getType() - == lookup.getFieldAccess(correlName).getField().getType(); - - assert pos != -1; - - if (bb.mapRootRelToFieldProjection.containsKey(bb.root)) { - // bb.root is an aggregate and only projects group by - // keys. - Map exprProjection = - bb.mapRootRelToFieldProjection.get(bb.root); - - // subquery can reference group by keys projected from - // the root of the outer relation. - if (exprProjection.containsKey(pos)) { - pos = exprProjection.get(pos); - } else { - // correl not grouped - throw Util.newInternal( - "Identifier '" + originalRelName + "." - + originalFieldName + "' is not a group expr"); - } - } - - Correlation newCorVar = - new Correlation( - getCorrelOrdinal(correlName), - pos); - - correlations.add(newCorVar); - } - } - - if (!correlations.isEmpty()) { - return new CorrelatorRel( - rightRel.getCluster(), - leftRel, - rightRel, - joinCond, - correlations, - joinType); - } - } - - final List extraLeftExprs = new ArrayList(); - final List extraRightExprs = new ArrayList(); - final int leftCount = leftRel.getRowType().getFieldCount(); - final int rightCount = rightRel.getRowType().getFieldCount(); - if (!containsGet(joinCond)) { - joinCond = pushDownJoinConditions( - joinCond, leftCount, rightCount, extraLeftExprs, extraRightExprs); - } - if (!extraLeftExprs.isEmpty()) { - final List fields = - leftRel.getRowType().getFieldList(); - leftRel = RelOptUtil.createProject( - leftRel, - new AbstractList>() { - @Override - public int size() { - return leftCount + extraLeftExprs.size(); - } - - @Override - public Pair get(int index) { - if (index < leftCount) { - RelDataTypeField field = fields.get(index); - return Pair.of( - new RexInputRef(index, field.getType()), - field.getName()); - } else { - return Pair.of( - extraLeftExprs.get(index - leftCount), null); - } - } - }, - true); - } - if (!extraRightExprs.isEmpty()) { - final List fields = - rightRel.getRowType().getFieldList(); - final int newLeftCount = leftCount + extraLeftExprs.size(); - rightRel = RelOptUtil.createProject( - rightRel, - new AbstractList>() { - @Override - public int size() { - return rightCount + extraRightExprs.size(); - } - - @Override - public Pair get(int index) { - if (index < rightCount) { - RelDataTypeField field = fields.get(index); - return Pair.of( - new RexInputRef(index, field.getType()), - field.getName()); - } else { - return Pair.of( - RexUtil.shift( - extraRightExprs.get(index - rightCount), - -newLeftCount), - null); - } - } - }, - true); - } - RelNode join = createJoin( - leftRel, - rightRel, - joinCond, - joinType, - ImmutableSet.of()); - if (!extraLeftExprs.isEmpty() || !extraRightExprs.isEmpty()) { - Mappings.TargetMapping mapping = - Mappings.createShiftMapping( - leftCount + extraLeftExprs.size() - + rightCount + extraRightExprs.size(), - 0, 0, leftCount, - leftCount, leftCount + extraLeftExprs.size(), rightCount); - return RelOptUtil.project(join, mapping); - } - return join; - } - - private static boolean containsGet(RexNode node) { - try { - node.accept( - new RexVisitorImpl(true) { - @Override public Void visitCall(RexCall call) { - if (call.getOperator() == RexBuilder.GET_OPERATOR) { - throw Util.FoundOne.NULL; - } - return super.visitCall(call); - } - }); - return false; - } catch (Util.FoundOne e) { - return true; - } - } - - /** - * Pushes down parts of a join condition. For example, given - * "emp JOIN dept ON emp.deptno + 1 = dept.deptno", adds a project above - * "emp" that computes the expression - * "emp.deptno + 1". The resulting join condition is a simple combination - * of AND, equals, and input fields. - */ - private RexNode pushDownJoinConditions( - RexNode node, - int leftCount, - int rightCount, - List extraLeftExprs, - List extraRightExprs) { - switch (node.getKind()) { - case AND: - case OR: - case EQUALS: - RexCall call = (RexCall) node; - List list = new ArrayList(); - List operands = Lists.newArrayList(call.getOperands()); - for (int i = 0; i < operands.size(); i++) { - RexNode operand = operands.get(i); - final int left2 = leftCount + extraLeftExprs.size(); - final int right2 = rightCount + extraRightExprs.size(); - final RexNode e = - pushDownJoinConditions( - operand, - leftCount, - rightCount, - extraLeftExprs, - extraRightExprs); - final List remainingOperands = Util.skip(operands, i + 1); - final int left3 = leftCount + extraLeftExprs.size(); - final int right3 = rightCount + extraRightExprs.size(); - fix(remainingOperands, left2, left3); - fix(list, left2, left3); - list.add(e); - } - if (!list.equals(call.getOperands())) { - return call.clone(call.getType(), list); - } - return call; - case INPUT_REF: - case LITERAL: - return node; - default: - BitSet bits = RelOptUtil.InputFinder.bits(node); - final int mid = leftCount + extraLeftExprs.size(); - switch (Side.of(bits, mid)) { - case LEFT: - fix(extraRightExprs, mid, mid + 1); - extraLeftExprs.add(node); - return new RexInputRef(mid, node.getType()); - case RIGHT: - final int index2 = mid + rightCount + extraRightExprs.size(); - extraRightExprs.add(node); - return new RexInputRef(index2, node.getType()); - case BOTH: - case EMPTY: - default: - return node; - } - } - } - - private void fix(List operands, int before, int after) { - if (before == after) { - return; - } - for (int i = 0; i < operands.size(); i++) { - RexNode node = operands.get(i); - operands.set(i, RexUtil.shift(node, before, after - before)); - } - } - - /** - * Categorizes whether a bit set contains bits left and right of a - * line. - */ - enum Side { - LEFT, RIGHT, BOTH, EMPTY; - - static Side of(BitSet bitSet, int middle) { - final int firstBit = bitSet.nextSetBit(0); - if (firstBit < 0) { - return EMPTY; - } - if (firstBit >= middle) { - return RIGHT; - } - if (bitSet.nextSetBit(middle) < 0) { - return LEFT; - } - return BOTH; - } - } - - /** - * Determines whether a subquery is non-correlated. Note that a - * non-correlated subquery can contain correlated references, provided those - * references do not reference select statements that are parents of the - * subquery. - * - * @param subq the subquery - * @param bb blackboard used while converting the subquery, i.e., the - * blackboard of the parent query of this subquery - * @return true if the subquery is non-correlated. - */ - private boolean isSubqNonCorrelated(RelNode subq, Blackboard bb) { - Set correlatedVariables = RelOptUtil.getVariablesUsed(subq); - for (String correlName : correlatedVariables) { - DeferredLookup lookup = mapCorrelToDeferred.get(correlName); - String originalRelName = lookup.getOriginalRelName(); - - int[] nsIndexes = {-1}; - final SqlValidatorScope[] ancestorScopes = {null}; - SqlValidatorNamespace foundNs = - lookup.bb.scope.resolve( - originalRelName, - ancestorScopes, - nsIndexes); - - assert foundNs != null; - assert nsIndexes.length == 1; - - SqlValidatorScope ancestorScope = ancestorScopes[0]; - - // If the correlated reference is in a scope that's "above" the - // subquery, then this is a correlated subquery. - SqlValidatorScope parentScope = bb.scope; - do { - if (ancestorScope == parentScope) { - return false; - } - if (parentScope instanceof DelegatingScope) { - parentScope = ((DelegatingScope) parentScope).getParent(); - } else { - break; - } - } while (parentScope != null); - } - return true; - } - - /** - * Returns a list of fields to be prefixed to each relational expression. - * - * @return List of system fields - */ - protected List getSystemFields() { - return Collections.emptyList(); - } - - private RexNode convertJoinCondition( - Blackboard bb, - SqlNode condition, - JoinConditionType conditionType, - RelNode leftRel, - RelNode rightRel) { - if (condition == null) { - return rexBuilder.makeLiteral(true); - } - bb.setRoot(ImmutableList.of(leftRel, rightRel)); - replaceSubqueries(bb, condition, RelOptUtil.Logic.UNKNOWN_AS_FALSE); - switch (conditionType) { - case ON: - bb.setRoot(ImmutableList.of(leftRel, rightRel)); - return bb.convertExpression(condition); - case USING: - SqlNodeList list = (SqlNodeList) condition; - List nameList = new ArrayList(); - for (SqlNode columnName : list) { - final SqlIdentifier id = (SqlIdentifier) columnName; - String name = id.getSimple(); - nameList.add(name); - } - return convertUsing(leftRel, rightRel, nameList); - default: - throw Util.unexpected(conditionType); - } - } - - /** - * Returns an expression for matching columns of a USING clause or inferred - * from NATURAL JOIN. "a JOIN b USING (x, y)" becomes "a.x = b.x AND a.y = - * b.y". Returns null if the column list is empty. - * - * @param leftRel Left input to the join - * @param rightRel Right input to the join - * @param nameList List of column names to join on - * @return Expression to match columns from name list, or true if name list - * is empty - */ - private RexNode convertUsing( - RelNode leftRel, - RelNode rightRel, - List nameList) { - final List list = Lists.newArrayList(); - for (String name : nameList) { - final RelDataType leftRowType = leftRel.getRowType(); - RelDataTypeField leftField = catalogReader.field(leftRowType, name); - RexNode left = - rexBuilder.makeInputRef( - leftField.getType(), - leftField.getIndex()); - final RelDataType rightRowType = rightRel.getRowType(); - RelDataTypeField rightField = - catalogReader.field(rightRowType, name); - RexNode right = - rexBuilder.makeInputRef( - rightField.getType(), - leftRowType.getFieldList().size() + rightField.getIndex()); - RexNode equalsCall = - rexBuilder.makeCall( - SqlStdOperatorTable.EQUALS, - left, - right); - list.add(equalsCall); - } - return RexUtil.composeConjunction(rexBuilder, list, false); - } - - private static JoinRelType convertJoinType(JoinType joinType) { - switch (joinType) { - case COMMA: - case INNER: - case CROSS: - return JoinRelType.INNER; - case FULL: - return JoinRelType.FULL; - case LEFT: - return JoinRelType.LEFT; - case RIGHT: - return JoinRelType.RIGHT; - default: - throw Util.unexpected(joinType); - } - } - - /** - * Converts the SELECT, GROUP BY and HAVING clauses of an aggregate query. - * - *

This method extracts SELECT, GROUP BY and HAVING clauses, and creates - * an {@link AggConverter}, then delegates to {@link #createAggImpl}. - * Derived class may override this method to change any of those clauses or - * specify a different {@link AggConverter}. - * - * @param bb Scope within which to resolve identifiers - * @param select Query - * @param orderExprList Additional expressions needed to implement ORDER BY - */ - protected void convertAgg( - Blackboard bb, - SqlSelect select, - List orderExprList) { - assert bb.root != null : "precondition: child != null"; - SqlNodeList groupList = select.getGroup(); - SqlNodeList selectList = select.getSelectList(); - SqlNode having = select.getHaving(); - - final AggConverter aggConverter = new AggConverter(bb, select); - createAggImpl( - bb, - aggConverter, - selectList, - groupList, - having, - orderExprList); - } - - protected final void createAggImpl( - Blackboard bb, - AggConverter aggConverter, - SqlNodeList selectList, - SqlNodeList groupList, - SqlNode having, - List orderExprList) { - SqlNodeList aggList = new SqlNodeList(SqlParserPos.ZERO); - - for (SqlNode selectNode : selectList) { - if (validator.isAggregate(selectNode)) { - aggList.add(selectNode); - } - } - - // first replace the subqueries inside the aggregates - // because they will provide input rows to the aggregates. - replaceSubqueries(bb, aggList, RelOptUtil.Logic.TRUE_FALSE_UNKNOWN); - - // If group-by clause is missing, pretend that it has zero elements. - if (groupList == null) { - groupList = SqlNodeList.EMPTY; - } - - // register the group exprs - - // build a map to remember the projections from the top scope to the - // output of the current root. - // - // Currently farrago allows expressions, not just column references in - // group by list. This is not SQL 2003 compliant. - - Map groupExprProjection = - new HashMap(); - - int i = -1; - for (SqlNode groupExpr : groupList) { - ++i; - final SqlNode expandedGroupExpr = - validator.expand(groupExpr, bb.scope); - aggConverter.addGroupExpr(expandedGroupExpr); - - if (expandedGroupExpr instanceof SqlIdentifier) { - // SQL 2003 does not allow expressions of column references - SqlIdentifier expr = (SqlIdentifier) expandedGroupExpr; - - // column references should be fully qualified. - assert expr.names.size() == 2; - String originalRelName = expr.names.get(0); - String originalFieldName = expr.names.get(1); - - int[] nsIndexes = {-1}; - final SqlValidatorScope[] ancestorScopes = {null}; - SqlValidatorNamespace foundNs = - bb.scope.resolve( - originalRelName, - ancestorScopes, - nsIndexes); - - assert foundNs != null; - assert nsIndexes.length == 1; - int childNamespaceIndex = nsIndexes[0]; - - int namespaceOffset = 0; - - if (childNamespaceIndex > 0) { - // If not the first child, need to figure out the width of - // output types from all the preceding namespaces - assert ancestorScopes[0] instanceof ListScope; - List children = - ((ListScope) ancestorScopes[0]).getChildren(); - - for (int j = 0; j < childNamespaceIndex; j++) { - namespaceOffset += - children.get(j).getRowType().getFieldCount(); - } - } - - RelDataTypeField field = - catalogReader.field(foundNs.getRowType(), originalFieldName); - int origPos = namespaceOffset + field.getIndex(); - - groupExprProjection.put(origPos, i); - } - } - - RexNode havingExpr = null; - List selectExprs = new ArrayList(); - List selectNames = new ArrayList(); - - try { - Util.permAssert(bb.agg == null, "already in agg mode"); - bb.agg = aggConverter; - - // convert the select and having expressions, so that the - // agg converter knows which aggregations are required - - selectList.accept(aggConverter); - for (SqlNode expr : orderExprList) { - expr.accept(aggConverter); - } - if (having != null) { - having.accept(aggConverter); - } - - // compute inputs to the aggregator - List preExprs = aggConverter.getPreExprs(); - List preNames = aggConverter.getPreNames(); - - if (preExprs.size() == 0) { - // Special case for COUNT(*), where we can end up with no inputs - // at all. The rest of the system doesn't like 0-tuples, so we - // select a dummy constant here. - preExprs = - Collections.singletonList( - (RexNode) rexBuilder.makeExactLiteral(BigDecimal.ZERO)); - preNames = Collections.singletonList(null); - } - - final RelNode inputRel = bb.root; - - // Project the expressions required by agg and having. - bb.setRoot( - RelOptUtil.createProject( - inputRel, - preExprs, - preNames, - true), - false); - bb.mapRootRelToFieldProjection.put(bb.root, groupExprProjection); - - // REVIEW jvs 31-Oct-2007: doesn't the declaration of - // monotonicity here assume sort-based aggregation at - // the physical level? - - // Tell bb which of group columns are sorted. - bb.columnMonotonicities.clear(); - for (SqlNode groupItem : groupList) { - bb.columnMonotonicities.add( - bb.scope.getMonotonicity(groupItem)); - } - - // Add the aggregator - bb.setRoot( - createAggregate( - bb, - BitSets.range(aggConverter.groupExprs.size()), - aggConverter.getAggCalls()), - false); - - bb.mapRootRelToFieldProjection.put(bb.root, groupExprProjection); - - // Replace subqueries in having here and modify having to use - // the replaced expressions - if (having != null) { - SqlNode newHaving = pushDownNotForIn(having); - replaceSubqueries(bb, newHaving, RelOptUtil.Logic.UNKNOWN_AS_FALSE); - havingExpr = bb.convertExpression(newHaving); - if (havingExpr.isAlwaysTrue()) { - havingExpr = null; - } - } - - // Now convert the other subqueries in the select list. - // This needs to be done separately from the subquery inside - // any aggregate in the select list, and after the aggregate rel - // is allocated. - replaceSubqueries(bb, selectList, RelOptUtil.Logic.TRUE_FALSE_UNKNOWN); - - // Now subqueries in the entire select list have been converted. - // Convert the select expressions to get the final list to be - // projected. - int k = 0; - - // For select expressions, use the field names previously assigned - // by the validator. If we derive afresh, we might generate names - // like "EXPR$2" that don't match the names generated by the - // validator. This is especially the case when there are system - // fields; system fields appear in the relnode's rowtype but do not - // (yet) appear in the validator type. - final SelectScope selectScope = - SqlValidatorUtil.getEnclosingSelectScope(bb.scope); - final SqlValidatorNamespace selectNamespace = - validator.getNamespace(selectScope.getNode()); - final List names = - selectNamespace.getRowType().getFieldNames(); - int sysFieldCount = selectList.size() - names.size(); - for (SqlNode expr : selectList) { - selectExprs.add(bb.convertExpression(expr)); - selectNames.add( - k < sysFieldCount - ? validator.deriveAlias(expr, k++) - : names.get(k++ - sysFieldCount)); - } - - for (SqlNode expr : orderExprList) { - selectExprs.add(bb.convertExpression(expr)); - selectNames.add(validator.deriveAlias(expr, k++)); - } - } finally { - bb.agg = null; - } - - // implement HAVING (we have already checked that it is non-trivial) - if (havingExpr != null) { - bb.setRoot(RelOptUtil.createFilter(bb.root, havingExpr), false); - } - - // implement the SELECT list - bb.setRoot( - RelOptUtil.createProject( - bb.root, - selectExprs, - selectNames, - true), - false); - - // Tell bb which of group columns are sorted. - bb.columnMonotonicities.clear(); - for (SqlNode selectItem : selectList) { - bb.columnMonotonicities.add( - bb.scope.getMonotonicity(selectItem)); - } - } - - /** - * Creates an AggregateRel. - * - *

In case the aggregate rel changes the order in which it projects - * fields, the groupExprProjection parameter is provided, and - * the implementation of this method may modify it. - * - *

The sortedCount parameter is the number of expressions - * known to be monotonic. These expressions must be on the leading edge of - * the grouping keys. The default implementation of this method ignores this - * parameter. - * - * @param bb Blackboard - * @param groupSet Bit set of ordinals of grouping columns - * @param aggCalls Array of calls to aggregate functions - * @return AggregateRel - */ - protected RelNode createAggregate( - Blackboard bb, - BitSet groupSet, - List aggCalls) { - return new AggregateRel( - cluster, - bb.root, - groupSet, - aggCalls); - } - - public RexDynamicParam convertDynamicParam( - final SqlDynamicParam dynamicParam) { - // REVIEW jvs 8-Jan-2005: dynamic params may be encountered out of - // order. Should probably cross-check with the count from the parser - // at the end and make sure they all got filled in. Why doesn't List - // have a resize() method?!? Make this a utility. - while (dynamicParam.getIndex() >= dynamicParamSqlNodes.size()) { - dynamicParamSqlNodes.add(null); - } - - dynamicParamSqlNodes.set( - dynamicParam.getIndex(), - dynamicParam); - return rexBuilder.makeDynamicParam( - getDynamicParamType(dynamicParam.getIndex()), - dynamicParam.getIndex()); - } - - /** - * Creates a list of collations required to implement the ORDER BY clause, - * if there is one. Populates extraOrderExprs with any sort - * expressions which are not in the select clause. - * - * @param bb Scope within which to resolve identifiers - * @param select Select clause. Never null, because we invent a - * dummy SELECT if ORDER BY is applied to a set - * operation (UNION etc.) - * @param orderList Order by clause, may be null - * @param extraOrderExprs Sort expressions which are not in the select - * clause (output) - * @param collationList List of collations (output) - */ - protected void gatherOrderExprs( - Blackboard bb, - SqlSelect select, - SqlNodeList orderList, - List extraOrderExprs, - List collationList) { - // TODO: add validation rules to SqlValidator also - assert bb.root != null : "precondition: child != null"; - assert select != null; - if (orderList == null) { - return; - } - for (SqlNode orderItem : orderList) { - collationList.add( - convertOrderItem( - select, - orderItem, - extraOrderExprs, - RelFieldCollation.Direction.ASCENDING, - RelFieldCollation.NullDirection.UNSPECIFIED)); - } - } - - protected RelFieldCollation convertOrderItem( - SqlSelect select, - SqlNode orderItem, List extraExprs, - RelFieldCollation.Direction direction, - RelFieldCollation.NullDirection nullDirection) { - assert select != null; - // Handle DESC keyword, e.g. 'select a, b from t order by a desc'. - switch (orderItem.getKind()) { - case DESCENDING: - return convertOrderItem( - select, - ((SqlCall) orderItem).operand(0), - extraExprs, - RelFieldCollation.Direction.DESCENDING, - nullDirection); - case NULLS_FIRST: - return convertOrderItem( - select, - ((SqlCall) orderItem).operand(0), - extraExprs, - direction, - RelFieldCollation.NullDirection.FIRST); - case NULLS_LAST: - return convertOrderItem( - select, - ((SqlCall) orderItem).operand(0), - extraExprs, - direction, - RelFieldCollation.NullDirection.LAST); - } - - SqlNode converted = validator.expandOrderExpr(select, orderItem); - - // Scan the select list and order exprs for an identical expression. - final SelectScope selectScope = validator.getRawSelectScope(select); - int ordinal = -1; - for (SqlNode selectItem : selectScope.getExpandedSelectList()) { - ++ordinal; - if (converted.equalsDeep(stripAs(selectItem), false)) { - return new RelFieldCollation( - ordinal, direction, nullDirection); - } - } - - for (SqlNode extraExpr : extraExprs) { - ++ordinal; - if (converted.equalsDeep(extraExpr, false)) { - return new RelFieldCollation( - ordinal, direction, nullDirection); - } - } - - // TODO: handle collation sequence - // TODO: flag expressions as non-standard - - extraExprs.add(converted); - return new RelFieldCollation(ordinal + 1, direction, nullDirection); - } - - protected boolean enableDecorrelation() { - // disable subquery decorrelation when needed. - // e.g. if outer joins are not supported. - return decorrelationEnabled; - } - - protected RelNode decorrelateQuery(RelNode rootRel) { - return RelDecorrelator.decorrelateQuery(rootRel); - } - - /** - * Sets whether to trim unused fields as part of the conversion process. - * - * @param trim Whether to trim unused fields - */ - public void setTrimUnusedFields(boolean trim) { - this.trimUnusedFields = trim; - } - - /** - * Returns whether to trim unused fields as part of the conversion process. - * - * @return Whether to trim unused fields - */ - public boolean isTrimUnusedFields() { - // OVERRIDE POINT - return false; // was `trimUnusedFields` - } - - /** - * Recursively converts a query to a relational expression. - * - * @param query Query - * @param top Whether this query is the top-level query of the - * statement - * @param targetRowType Target row type, or null - * @return Relational expression - */ - protected RelNode convertQueryRecursive( - SqlNode query, - boolean top, - RelDataType targetRowType) { - switch (query.getKind()) { - case SELECT: - return convertSelect((SqlSelect) query); - case INSERT: - return convertInsert((SqlInsert) query); - case DELETE: - return convertDelete((SqlDelete) query); - case UPDATE: - return convertUpdate((SqlUpdate) query); - case MERGE: - return convertMerge((SqlMerge) query); - case UNION: - case INTERSECT: - case EXCEPT: - return convertSetOp((SqlCall) query); - case WITH: - return convertWith((SqlWith) query); - case VALUES: - return convertValues((SqlCall) query, targetRowType); - default: - throw Util.newInternal("not a query: " + query); - } - } - - /** - * Converts a set operation (UNION, INTERSECT, MINUS) into relational - * expressions. - * - * @param call Call to set operator - * @return Relational expression - */ - protected RelNode convertSetOp(SqlCall call) { - final RelNode left = convertQueryRecursive(call.operand(0), false, null); - final RelNode right = convertQueryRecursive(call.operand(1), false, null); - boolean all = false; - if (call.getOperator() instanceof SqlSetOperator) { - all = ((SqlSetOperator) (call.getOperator())).isAll(); - } - switch (call.getKind()) { - case UNION: - return new UnionRel( - cluster, - ImmutableList.of(left, right), - all); - - case INTERSECT: - // TODO: all - if (!all) { - return new IntersectRel( - cluster, - ImmutableList.of(left, right), - all); - } else { - throw Util.newInternal( - "set operator INTERSECT ALL not suported"); - } - - case EXCEPT: - // TODO: all - if (!all) { - return new MinusRel( - cluster, - ImmutableList.of(left, right), - all); - } else { - throw Util.newInternal( - "set operator EXCEPT ALL not suported"); - } - - default: - throw Util.unexpected(call.getKind()); - } - } - - protected RelNode convertInsert(SqlInsert call) { - RelOptTable targetTable = getTargetTable(call); - - final RelDataType targetRowType = - validator.getValidatedNodeType(call); - assert targetRowType != null; - RelNode sourceRel = - convertQueryRecursive( - call.getSource(), - false, - targetRowType); - RelNode massagedRel = convertColumnList(call, sourceRel); - - final ModifiableTable modifiableTable = - targetTable.unwrap(ModifiableTable.class); - if (modifiableTable != null) { - return modifiableTable.toModificationRel( - cluster, - targetTable, - catalogReader, - massagedRel, - TableModificationRel.Operation.INSERT, - null, - false); - } - return new TableModificationRel( - cluster, - targetTable, - catalogReader, - massagedRel, - TableModificationRel.Operation.INSERT, - null, - false); - } - - private RelOptTable.ToRelContext createToRelContext() { - return new RelOptTable.ToRelContext() { - public RelOptCluster getCluster() { - return cluster; - } - - public RelNode expandView( - RelDataType rowType, - String queryString, - List schemaPath) { - return viewExpander.expandView(rowType, queryString, schemaPath); - } - }; - } - - public RelNode toRel(RelOptTable table) { - return table.toRel(createToRelContext()); - } - - protected RelOptTable getTargetTable(SqlNode call) { - SqlValidatorNamespace targetNs = validator.getNamespace(call).resolve(); - return SqlValidatorUtil.getRelOptTable(targetNs, catalogReader, null, null); - } - - /** - * Creates a source for an INSERT statement. - * - *

If the column list is not specified, source expressions match target - * columns in order. - * - *

If the column list is specified, Source expressions are mapped to - * target columns by name via targetColumnList, and may not cover the entire - * target table. So, we'll make up a full row, using a combination of - * default values and the source expressions provided. - * - * @param call Insert expression - * @param sourceRel Source relational expression - * @return Converted INSERT statement - */ - protected RelNode convertColumnList( - SqlInsert call, - RelNode sourceRel) { - RelDataType sourceRowType = sourceRel.getRowType(); - final RexNode sourceRef = - rexBuilder.makeRangeReference(sourceRowType, 0, false); - final List targetColumnNames = new ArrayList(); - final List columnExprs = new ArrayList(); - collectInsertTargets(call, sourceRef, targetColumnNames, columnExprs); - - final RelOptTable targetTable = getTargetTable(call); - final RelDataType targetRowType = targetTable.getRowType(); - final List targetFields = - targetRowType.getFieldList(); - final List sourceExps = - new ArrayList( - Collections.nCopies(targetFields.size(), null)); - final List fieldNames = - new ArrayList( - Collections.nCopies(targetFields.size(), null)); - - // Walk the name list and place the associated value in the - // expression list according to the ordinal value returned from - // the table construct, leaving nulls in the list for columns - // that are not referenced. - for (Pair p : Pair.zip(targetColumnNames, columnExprs)) { - RelDataTypeField field = catalogReader.field(targetRowType, p.left); - assert field != null : "column " + p.left + " not found"; - sourceExps.set(field.getIndex(), p.right); - } - - // Walk the expression list and get default values for any columns - // that were not supplied in the statement. Get field names too. - for (int i = 0; i < targetFields.size(); ++i) { - final RelDataTypeField field = targetFields.get(i); - final String fieldName = field.getName(); - fieldNames.set(i, fieldName); - if (sourceExps.get(i) != null) { - if (defaultValueFactory.isGeneratedAlways(targetTable, i)) { - throw RESOURCE.insertIntoAlwaysGenerated(fieldName).ex(); - } - continue; - } - sourceExps.set( - i, defaultValueFactory.newColumnDefaultValue(targetTable, i)); - - // bare nulls are dangerous in the wrong hands - sourceExps.set( - i, - castNullLiteralIfNeeded( - sourceExps.get(i), field.getType())); - } - - return RelOptUtil.createProject(sourceRel, sourceExps, fieldNames, true); - } - - private RexNode castNullLiteralIfNeeded(RexNode node, RelDataType type) { - if (!RexLiteral.isNullLiteral(node)) { - return node; - } - return rexBuilder.makeCast(type, node); - } - - /** - * Given an INSERT statement, collects the list of names to be populated and - * the expressions to put in them. - * - * @param call Insert statement - * @param sourceRef Expression representing a row from the source - * relational expression - * @param targetColumnNames List of target column names, to be populated - * @param columnExprs List of expressions, to be populated - */ - protected void collectInsertTargets( - SqlInsert call, - final RexNode sourceRef, - final List targetColumnNames, - List columnExprs) { - final RelOptTable targetTable = getTargetTable(call); - final RelDataType targetRowType = targetTable.getRowType(); - SqlNodeList targetColumnList = call.getTargetColumnList(); - if (targetColumnList == null) { - targetColumnNames.addAll(targetRowType.getFieldNames()); - } else { - for (int i = 0; i < targetColumnList.size(); i++) { - SqlIdentifier id = (SqlIdentifier) targetColumnList.get(i); - targetColumnNames.add(id.getSimple()); - } - } - - for (int i = 0; i < targetColumnNames.size(); i++) { - final RexNode expr = rexBuilder.makeFieldAccess(sourceRef, i); - columnExprs.add(expr); - } - } - - private RelNode convertDelete(SqlDelete call) { - RelOptTable targetTable = getTargetTable(call); - RelNode sourceRel = convertSelect(call.getSourceSelect()); - return new TableModificationRel( - cluster, - targetTable, - catalogReader, - sourceRel, - TableModificationRel.Operation.DELETE, - null, - false); - } - - private RelNode convertUpdate(SqlUpdate call) { - RelOptTable targetTable = getTargetTable(call); - - // convert update column list from SqlIdentifier to String - List targetColumnNameList = new ArrayList(); - for (SqlNode node : call.getTargetColumnList()) { - SqlIdentifier id = (SqlIdentifier) node; - String name = id.getSimple(); - targetColumnNameList.add(name); - } - - RelNode sourceRel = convertSelect(call.getSourceSelect()); - - return new TableModificationRel( - cluster, - targetTable, - catalogReader, - sourceRel, - TableModificationRel.Operation.UPDATE, - targetColumnNameList, - false); - } - - private RelNode convertMerge(SqlMerge call) { - RelOptTable targetTable = getTargetTable(call); - - // convert update column list from SqlIdentifier to String - List targetColumnNameList = new ArrayList(); - SqlUpdate updateCall = call.getUpdateCall(); - if (updateCall != null) { - for (SqlNode targetColumn : updateCall.getTargetColumnList()) { - SqlIdentifier id = (SqlIdentifier) targetColumn; - String name = id.getSimple(); - targetColumnNameList.add(name); - } - } - - // replace the projection of the source select with a - // projection that contains the following: - // 1) the expressions corresponding to the new insert row (if there is - // an insert) - // 2) all columns from the target table (if there is an update) - // 3) the set expressions in the update call (if there is an update) - - // first, convert the merge's source select to construct the columns - // from the target table and the set expressions in the update call - RelNode mergeSourceRel = convertSelect(call.getSourceSelect()); - - // then, convert the insert statement so we can get the insert - // values expressions - SqlInsert insertCall = call.getInsertCall(); - int nLevel1Exprs = 0; - List level1InsertExprs = null; - List level2InsertExprs = null; - if (insertCall != null) { - RelNode insertRel = convertInsert(insertCall); - - // if there are 2 level of projections in the insert source, combine - // them into a single project; level1 refers to the topmost project; - // the level1 projection contains references to the level2 - // expressions, except in the case where no target expression was - // provided, in which case, the expression is the default value for - // the column; or if the expressions directly map to the source - // table - level1InsertExprs = - ((ProjectRel) insertRel.getInput(0)).getProjects(); - if (insertRel.getInput(0).getInput(0) instanceof ProjectRel) { - level2InsertExprs = - ((ProjectRel) insertRel.getInput(0).getInput(0)) - .getProjects(); - } - nLevel1Exprs = level1InsertExprs.size(); - } - - JoinRel joinRel = (JoinRel) mergeSourceRel.getInput(0); - int nSourceFields = joinRel.getLeft().getRowType().getFieldCount(); - List projects = new ArrayList(); - for (int level1Idx = 0; level1Idx < nLevel1Exprs; level1Idx++) { - if ((level2InsertExprs != null) - && (level1InsertExprs.get(level1Idx) instanceof RexInputRef)) { - int level2Idx = - ((RexInputRef) level1InsertExprs.get(level1Idx)).getIndex(); - projects.add(level2InsertExprs.get(level2Idx)); - } else { - projects.add(level1InsertExprs.get(level1Idx)); - } - } - if (updateCall != null) { - final ProjectRel project = (ProjectRel) mergeSourceRel; - projects.addAll( - Util.skip(project.getProjects(), nSourceFields)); - } - - RelNode massagedRel = - RelOptUtil.createProject(joinRel, projects, null, true); - - return new TableModificationRel( - cluster, - targetTable, - catalogReader, - massagedRel, - TableModificationRel.Operation.MERGE, - targetColumnNameList, - false); - } - - /** - * Converts an identifier into an expression in a given scope. For example, - * the "empno" in "select empno from emp join dept" becomes "emp.empno". - */ - private RexNode convertIdentifier( - Blackboard bb, - SqlIdentifier identifier) { - // first check for reserved identifiers like CURRENT_USER - final SqlCall call = SqlUtil.makeCall(opTab, identifier); - if (call != null) { - return bb.convertExpression(call); - } - - if (bb.agg != null) { - throw Util.newInternal("Identifier '" + identifier - + "' is not a group expr"); - } - - SqlValidatorNamespace namespace = null; - if (bb.scope != null) { - identifier = bb.scope.fullyQualify(identifier); - namespace = bb.scope.resolve(identifier.names.get(0), null, null); - } - RexNode e = bb.lookupExp(identifier.names.get(0)); - final String correlationName; - if (e instanceof RexCorrelVariable) { - correlationName = ((RexCorrelVariable) e).getName(); - } else { - correlationName = null; - } - - for (String name : Util.skip(identifier.names)) { - if (namespace != null) { - name = namespace.translate(name); - namespace = null; - } - final boolean caseSensitive = true; // name already fully-qualified - e = rexBuilder.makeFieldAccess(e, name, caseSensitive); - } - if (e instanceof RexInputRef) { - // adjust the type to account for nulls introduced by outer joins - e = adjustInputRef(bb, (RexInputRef) e); - } - - if (null != correlationName) { - // REVIEW: make mapCorrelateVariableToRexNode map to RexFieldAccess - assert e instanceof RexFieldAccess; - final RexNode prev = - bb.mapCorrelateVariableToRexNode.put(correlationName, e); - assert prev == null; - } - return e; - } - - /** - * Adjusts the type of a reference to an input field to account for nulls - * introduced by outer joins; and adjusts the offset to match the physical - * implementation. - * - * @param bb Blackboard - * @param inputRef Input ref - * @return Adjusted input ref - */ - protected RexNode adjustInputRef( - Blackboard bb, - RexInputRef inputRef) { - RelDataTypeField field = bb.getRootField(inputRef); - if (field != null) { - return rexBuilder.makeInputRef( - field.getType(), - inputRef.getIndex()); - } - return inputRef; - } - - /** - * Converts a row constructor into a relational expression. - * - * @param bb Blackboard - * @param rowConstructor Row constructor expression - * @return Relational expression which returns a single row. - * @pre isRowConstructor(rowConstructor) - */ - private RelNode convertRowConstructor( - Blackboard bb, - SqlCall rowConstructor) { - assert isRowConstructor(rowConstructor) : rowConstructor; - final List operands = rowConstructor.getOperandList(); - return convertMultisets(operands, bb); - } - - private RelNode convertCursor(Blackboard bb, SubQuery subQuery) { - final SqlCall cursorCall = (SqlCall) subQuery.node; - assert cursorCall.operandCount() == 1; - SqlNode query = cursorCall.operand(0); - RelNode converted = convertQuery(query, false, false); - int iCursor = bb.cursors.size(); - bb.cursors.add(converted); - subQuery.expr = - new RexInputRef( - iCursor, - converted.getRowType()); - return converted; - } - - private RelNode convertMultisets(final List operands, - Blackboard bb) { - // NOTE: Wael 2/04/05: this implementation is not the most efficient in - // terms of planning since it generates XOs that can be reduced. - List joinList = new ArrayList(); - List lastList = new ArrayList(); - for (int i = 0; i < operands.size(); i++) { - SqlNode operand = operands.get(i); - if (!(operand instanceof SqlCall)) { - lastList.add(operand); - continue; - } - - final SqlCall call = (SqlCall) operand; - final SqlOperator op = call.getOperator(); - if ((op != SqlStdOperatorTable.MULTISET_VALUE) - && (op != SqlStdOperatorTable.MULTISET_QUERY)) { - lastList.add(operand); - continue; - } - final RelNode input; - if (op == SqlStdOperatorTable.MULTISET_VALUE) { - final SqlNodeList list = - new SqlNodeList(call.getOperandList(), call.getParserPosition()); -// assert bb.scope instanceof SelectScope : bb.scope; - CollectNamespace nss = - (CollectNamespace) validator.getNamespace(call); - Blackboard usedBb; - if (null != nss) { - usedBb = createBlackboard(nss.getScope(), null); - } else { - usedBb = - createBlackboard( - new ListScope(bb.scope) { - public SqlNode getNode() { - return call; - } - }, - null); - } - RelDataType multisetType = validator.getValidatedNodeType(call); - validator.setValidatedNodeType( - list, - multisetType.getComponentType()); - input = convertQueryOrInList(usedBb, list); - } else { - input = convertQuery(call.operand(0), false, true); - } - - if (lastList.size() > 0) { - joinList.add(lastList); - } - lastList = new ArrayList(); - CollectRel collectRel = - new CollectRel( - cluster, - cluster.traitSetOf(Convention.NONE), - input, - validator.deriveAlias(call, i)); - joinList.add(collectRel); - } - - if (joinList.size() == 0) { - joinList.add(lastList); - } - - for (int i = 0; i < joinList.size(); i++) { - Object o = joinList.get(i); - if (o instanceof List) { - List projectList = (List) o; - final List selectList = new ArrayList(); - final List fieldNameList = new ArrayList(); - for (int j = 0; j < projectList.size(); j++) { - SqlNode operand = projectList.get(j); - selectList.add(bb.convertExpression(operand)); - - // REVIEW angel 5-June-2005: Use deriveAliasFromOrdinal - // instead of deriveAlias to match field names from - // SqlRowOperator. Otherwise, get error Type - // 'RecordType(INTEGER EMPNO)' has no field 'EXPR$0' when - // doing select * from unnest( select multiset[empno] - // from sales.emps); - - fieldNameList.add(SqlUtil.deriveAliasFromOrdinal(j)); - } - - RelNode projRel = - RelOptUtil.createProject( - new OneRowRel(cluster), - selectList, - fieldNameList); - - joinList.set(i, projRel); - } - } - - RelNode ret = (RelNode) joinList.get(0); - for (int i = 1; i < joinList.size(); i++) { - RelNode relNode = (RelNode) joinList.get(i); - ret = - createJoin( - ret, - relNode, - rexBuilder.makeLiteral(true), - JoinRelType.INNER, - ImmutableSet.of()); - } - return ret; - } - - /** - * Factory method that creates a join. - * A subclass can override to use a different kind of join. - * - * @param left Left input - * @param right Right input - * @param condition Join condition - * @param joinType Join type - * @param variablesStopped Set of names of variables which are set by the - * LHS and used by the RHS and are not available to - * nodes above this JoinRel in the tree - * @return A relational expression representing a join - */ - protected RelNode createJoin( - RelNode left, - RelNode right, - RexNode condition, - JoinRelType joinType, - Set variablesStopped) { - return new JoinRel( - cluster, - left, - right, - condition, - joinType, - variablesStopped); - } - - private void convertSelectList( - Blackboard bb, - SqlSelect select, - List orderList) { - SqlNodeList selectList = select.getSelectList(); - selectList = validator.expandStar(selectList, select, false); - - replaceSubqueries(bb, selectList, RelOptUtil.Logic.TRUE_FALSE_UNKNOWN); - - List fieldNames = new ArrayList(); - List exprs = new ArrayList(); - Collection aliases = new TreeSet(); - - // Project any system fields. (Must be done before regular select items, - // because offsets may be affected.) - final List columnMonotonicityList = - new ArrayList(); - extraSelectItems( - bb, - select, - exprs, - fieldNames, - aliases, - columnMonotonicityList); - - // Project select clause. - int i = -1; - for (SqlNode expr : selectList) { - ++i; - exprs.add(bb.convertExpression(expr)); - fieldNames.add(deriveAlias(expr, aliases, i)); - } - - // Project extra fields for sorting. - for (SqlNode expr : orderList) { - ++i; - SqlNode expr2 = validator.expandOrderExpr(select, expr); - exprs.add(bb.convertExpression(expr2)); - fieldNames.add(deriveAlias(expr, aliases, i)); - } - - fieldNames = SqlValidatorUtil.uniquify(fieldNames); - - RelNode inputRel = bb.root; - bb.setRoot( - RelOptUtil.createProject(bb.root, exprs, fieldNames), - false); - - assert bb.columnMonotonicities.isEmpty(); - bb.columnMonotonicities.addAll(columnMonotonicityList); - for (SqlNode selectItem : selectList) { - bb.columnMonotonicities.add( - selectItem.getMonotonicity(bb.scope)); - } - } - - /** - * Adds extra select items. The default implementation adds nothing; derived - * classes may add columns to exprList, nameList, aliasList and - * columnMonotonicityList. - * - * @param bb Blackboard - * @param select Select statement being translated - * @param exprList List of expressions in select clause - * @param nameList List of names, one per column - * @param aliasList Collection of aliases that have been used - * already - * @param columnMonotonicityList List of monotonicity, one per column - */ - protected void extraSelectItems( - Blackboard bb, - SqlSelect select, - List exprList, - List nameList, - Collection aliasList, - List columnMonotonicityList) { - } - - private String deriveAlias( - final SqlNode node, - Collection aliases, - final int ordinal) { - String alias = validator.deriveAlias(node, ordinal); - if ((alias == null) || aliases.contains(alias)) { - String aliasBase = (alias == null) ? "EXPR$" : alias; - for (int j = 0;; j++) { - alias = aliasBase + j; - if (!aliases.contains(alias)) { - break; - } - } - } - aliases.add(alias); - return alias; - } - - /** - * Converts a WITH sub-query into a relational expression. - */ - public RelNode convertWith(SqlWith with) { - return convertQuery(with.body, false, false); - } - - /** - * Converts a SELECT statement's parse tree into a relational expression. - */ - public RelNode convertValues( - SqlCall values, - RelDataType targetRowType) { - final SqlValidatorScope scope = validator.getOverScope(values); - assert scope != null; - final Blackboard bb = createBlackboard(scope, null); - convertValuesImpl(bb, values, targetRowType); - return bb.root; - } - - /** - * Converts a values clause (as in "INSERT INTO T(x,y) VALUES (1,2)") into a - * relational expression. - * - * @param bb Blackboard - * @param values Call to SQL VALUES operator - * @param targetRowType Target row type - */ - private void convertValuesImpl( - Blackboard bb, - SqlCall values, - RelDataType targetRowType) { - // Attempt direct conversion to ValuesRel; if that fails, deal with - // fancy stuff like subqueries below. - RelNode valuesRel = - convertRowValues( - bb, - values, - values.getOperandList(), - true, - targetRowType); - if (valuesRel != null) { - bb.setRoot(valuesRel, true); - return; - } - - List unionRels = new ArrayList(); - for (SqlNode rowConstructor1 : values.getOperandList()) { - SqlCall rowConstructor = (SqlCall) rowConstructor1; - Blackboard tmpBb = createBlackboard(bb.scope, null); - replaceSubqueries(tmpBb, rowConstructor, - RelOptUtil.Logic.TRUE_FALSE_UNKNOWN); - List> exps = - new ArrayList>(); - for (Ord operand : Ord.zip(rowConstructor.getOperandList())) { - exps.add( - Pair.of( - tmpBb.convertExpression(operand.e), - validator.deriveAlias(operand.e, operand.i))); - } - RelNode in = - (null == tmpBb.root) - ? new OneRowRel(cluster) - : tmpBb.root; - unionRels.add( - RelOptUtil.createProject( - in, - Pair.left(exps), - Pair.right(exps), - true)); - } - - if (unionRels.size() == 0) { - throw Util.newInternal("empty values clause"); - } else if (unionRels.size() == 1) { - bb.setRoot( - unionRels.get(0), - true); - } else { - bb.setRoot( - new UnionRel( - cluster, - unionRels, - true), - true); - } - - // REVIEW jvs 22-Jan-2004: should I add - // mapScopeToLux.put(validator.getScope(values),bb.root); - // ? - } - - private String createCorrel() { - int n = nextCorrel++; - return CORREL_PREFIX + n; - } - - private int getCorrelOrdinal(String correlName) { - assert correlName.startsWith(CORREL_PREFIX); - return Integer.parseInt(correlName.substring(CORREL_PREFIX.length())); - } - - //~ Inner Classes ---------------------------------------------------------- - - /** - * Workspace for translating an individual SELECT statement (or sub-SELECT). - */ - protected class Blackboard implements SqlRexContext, SqlVisitor { - /** - * Collection of {@link RelNode} objects which correspond to a SELECT - * statement. - */ - public final SqlValidatorScope scope; - private final Map nameToNodeMap; - public RelNode root; - private List inputs; - private final Map mapCorrelateVariableToRexNode = - new HashMap(); - - List cursors; - - /** - * List of IN and EXISTS nodes inside this - * SELECT statement (but not inside sub-queries). - */ - private final Set subqueryList = Sets.newLinkedHashSet(); - - private final Map subqueryMap = - Util.asIndexMap(subqueryList, FN); - - private boolean subqueryNeedsOuterJoin; - - /** - * Workspace for building aggregates. - */ - AggConverter agg; - - /** - * When converting window aggregate, we need to know if the window is - * guaranteed to be non-empty. - */ - SqlWindow window; - - /** - * Project the groupby expressions out of the root of this sub-select. - * Subqueries can reference group by expressions projected from the - * "right" to the subquery. - */ - private final Map> - mapRootRelToFieldProjection = - new HashMap>(); - - private final List columnMonotonicities = - new ArrayList(); - - private final List systemFieldList = - new ArrayList(); - - /** - * Creates a Blackboard. - * - * @param scope Name-resolution scope for expressions validated - * within this query. Can be null if this Blackboard is - * for a leaf node, say - * @param nameToNodeMap Map which translates the expression to map a - * given parameter into, if translating expressions; - * null otherwise - */ - protected Blackboard( - SqlValidatorScope scope, - Map nameToNodeMap) { - this.scope = scope; - this.nameToNodeMap = nameToNodeMap; - this.cursors = new ArrayList(); - subqueryNeedsOuterJoin = false; - } - - public RexNode register( - RelNode rel, - JoinRelType joinType) { - return register(rel, joinType, null); - } - - /** - * Registers a relational expression. - * - * @param rel Relational expression - * @param joinType Join type - * @param leftKeys LHS of IN clause, or null for expressions - * other than IN - * @return Expression with which to refer to the row (or partial row) - * coming from this relational expression's side of the join - */ - public RexNode register( - RelNode rel, - JoinRelType joinType, - List leftKeys) { - assert joinType != null; - if (root == null) { - assert leftKeys == null; - setRoot(rel, false); - return rexBuilder.makeRangeReference( - root.getRowType(), - 0, - false); - } - - final RexNode joinCond; - final int origLeftInputCount = root.getRowType().getFieldCount(); - if (leftKeys != null) { - List newLeftInputExpr = Lists.newArrayList(); - for (int i = 0; i < origLeftInputCount; i++) { - newLeftInputExpr.add(rexBuilder.makeInputRef(root, i)); - } - - final List leftJoinKeys = Lists.newArrayList(); - for (RexNode leftKey : leftKeys) { - newLeftInputExpr.add(leftKey); - leftJoinKeys.add(origLeftInputCount + leftJoinKeys.size()); - } - - ProjectRel newLeftInput = - (ProjectRel) RelOptUtil.createProject( - root, - newLeftInputExpr, - null, - true); - - // maintain the group by mapping in the new ProjectRel - if (mapRootRelToFieldProjection.containsKey(root)) { - mapRootRelToFieldProjection.put( - newLeftInput, - mapRootRelToFieldProjection.get(root)); - } - - setRoot(newLeftInput, false); - - // right fields appear after the LHS fields. - final int rightOffset = root.getRowType().getFieldCount() - - newLeftInput.getRowType().getFieldCount(); - final List rightKeys = - Util.range(rightOffset, rightOffset + leftJoinKeys.size()); - - joinCond = - RelOptUtil.createEquiJoinCondition(newLeftInput, leftJoinKeys, - rel, rightKeys, rexBuilder); - } else { - joinCond = rexBuilder.makeLiteral(true); - } - - int leftFieldCount = root.getRowType().getFieldCount(); - final RelNode join = - createJoin( - this, - root, - rel, - joinCond, - joinType); - - setRoot(join, false); - - if (leftKeys != null - && joinType == JoinRelType.LEFT) { - final int leftKeyCount = leftKeys.size(); - int rightFieldLength = rel.getRowType().getFieldCount(); - assert leftKeyCount == rightFieldLength - 1; - - final int rexRangeRefLength = leftKeyCount + rightFieldLength; - RelDataType returnType = - typeFactory.createStructType( - new AbstractList>() { - public Map.Entry get( - int index) { - return join.getRowType().getFieldList() - .get(origLeftInputCount + index); - } - - public int size() { - return rexRangeRefLength; - } - }); - - return rexBuilder.makeRangeReference( - returnType, - origLeftInputCount, - false); - } else { - return rexBuilder.makeRangeReference( - rel.getRowType(), - leftFieldCount, - joinType.generatesNullsOnRight()); - } - } - - /** - * Sets a new root relational expression, as the translation process - * backs its way further up the tree. - * - * @param root New root relational expression - * @param leaf Whether the relational expression is a leaf, that is, - * derived from an atomic relational expression such as a table - * name in the from clause, or the projection on top of a - * select-subquery. In particular, relational expressions - * derived from JOIN operators are not leaves, but set - * expressions are. - */ - public void setRoot(RelNode root, boolean leaf) { - setRoot( - Collections.singletonList(root), root, root instanceof JoinRel); - if (leaf) { - leaves.add(root); - } - this.columnMonotonicities.clear(); - } - - private void setRoot( - List inputs, - RelNode root, - boolean hasSystemFields) { - this.inputs = inputs; - this.root = root; - this.systemFieldList.clear(); - if (hasSystemFields) { - this.systemFieldList.addAll(getSystemFields()); - } - } - - /** - * Notifies this Blackboard that the root just set using {@link - * #setRoot(RelNode, boolean)} was derived using dataset substitution. - * - *

The default implementation is not interested in such - * notifications, and does nothing. - * - * @param datasetName Dataset name - */ - public void setDataset(String datasetName) { - } - - void setRoot(List inputs) { - setRoot(inputs, null, false); - } - - /** - * Returns an expression with which to reference a from-list item. - * - * @param name the alias of the from item - * @return a {@link RexFieldAccess} or {@link RexRangeRef}, or null if - * not found - */ - RexNode lookupExp(String name) { - if (nameToNodeMap != null) { - RexNode node = nameToNodeMap.get(name); - if (node == null) { - throw Util.newInternal( - "Unknown identifier '" + name - + "' encountered while expanding expression" + node); - } - return node; - } - int[] offsets = {-1}; - final SqlValidatorScope[] ancestorScopes = {null}; - SqlValidatorNamespace foundNs = - scope.resolve(name, ancestorScopes, offsets); - if (foundNs == null) { - return null; - } - - // Found in current query's from list. Find which from item. - // We assume that the order of the from clause items has been - // preserved. - SqlValidatorScope ancestorScope = ancestorScopes[0]; - boolean isParent = ancestorScope != scope; - if ((inputs != null) && !isParent) { - int offset = offsets[0]; - final LookupContext rels = - new LookupContext(this, inputs, systemFieldList.size()); - return lookup(offset, rels); - } else { - // We're referencing a relational expression which has not been - // converted yet. This occurs when from items are correlated, - // e.g. "select from emp as emp join emp.getDepts() as dept". - // Create a temporary expression. - assert isParent; - DeferredLookup lookup = new DeferredLookup(this, name); - String correlName = createCorrel(); - mapCorrelToDeferred.put(correlName, lookup); - final RelDataType rowType = foundNs.getRowType(); - return rexBuilder.makeCorrel(rowType, correlName); - } - } - - /** - * Creates an expression with which to reference the expression whose - * offset in its from-list is {@code offset}. - */ - RexNode lookup( - int offset, - LookupContext lookupContext) { - Pair pair = lookupContext.findRel(offset); - return rexBuilder.makeRangeReference( - pair.left.getRowType(), - pair.right, - false); - } - - RelDataTypeField getRootField(RexInputRef inputRef) { - int fieldOffset = inputRef.getIndex(); - for (RelNode input : inputs) { - RelDataType rowType = input.getRowType(); - if (rowType == null) { - // TODO: remove this once leastRestrictive - // is correctly implemented - return null; - } - if (fieldOffset < rowType.getFieldCount()) { - return rowType.getFieldList().get(fieldOffset); - } - fieldOffset -= rowType.getFieldCount(); - } - throw new AssertionError(); - } - - public void flatten( - List rels, - int systemFieldCount, - int[] start, - List> relOffsetList) { - for (RelNode rel : rels) { - if (leaves.contains(rel)) { - relOffsetList.add( - Pair.of(rel, start[0])); - start[0] += rel.getRowType().getFieldCount(); - } else { - if (rel instanceof JoinRel - || rel instanceof AggregateRel) { - start[0] += systemFieldCount; - } - flatten( - rel.getInputs(), - systemFieldCount, - start, - relOffsetList); - } - } - } - - void registerSubquery(SqlNode node, RelOptUtil.Logic logic) { - subqueryList.add(new SubQuery(node, logic)); - } - - ImmutableList retrieveCursors() { - try { - return ImmutableList.copyOf(cursors); - } finally { - cursors.clear(); - } - } - - // implement SqlRexContext - public RexNode convertExpression(SqlNode expr) { - // If we're in aggregation mode and this is an expression in the - // GROUP BY clause, return a reference to the field. - if (agg != null) { - final SqlNode expandedGroupExpr = validator.expand(expr, scope); - RexNode rex = agg.lookupGroupExpr(expandedGroupExpr); - if (rex != null) { - return rex; - } - if (expr instanceof SqlCall) { - rex = agg.lookupAggregates((SqlCall) expr); - if (rex != null) { - return rex; - } - } - } - - // Allow the derived class chance to override the standard - // behavior for special kinds of expressions. - RexNode rex = convertExtendedExpression(expr, this); - if (rex != null) { - return rex; - } - - boolean needTruthTest; - - // Sub-queries and OVER expressions are not like ordinary - // expressions. - final SqlKind kind = expr.getKind(); - final SubQuery subQuery; - switch (kind) { - case CURSOR: - case SELECT: - case EXISTS: - case SCALAR_QUERY: - subQuery = subqueryMap.get(expr); - assert subQuery != null; - rex = subQuery.expr; - assert rex != null : "rex != null"; - - if (kind == SqlKind.CURSOR) { - // cursor reference is pre-baked - return rex; - } - if (((kind == SqlKind.SCALAR_QUERY) - || (kind == SqlKind.EXISTS)) - && isConvertedSubq(rex)) { - // scalar subquery or EXISTS has been converted to a - // constant - return rex; - } - - RexNode fieldAccess; - needTruthTest = false; - - // The indicator column is the last field of the subquery. - fieldAccess = - rexBuilder.makeFieldAccess( - rex, - rex.getType().getFieldCount() - 1); - - // The indicator column will be nullable if it comes from - // the null-generating side of the join. For EXISTS, add an - // "IS TRUE" check so that the result is "BOOLEAN NOT NULL". - if (fieldAccess.getType().isNullable()) { - if (kind == SqlKind.EXISTS) { - needTruthTest = true; - } - } - - if (needTruthTest) { - fieldAccess = - rexBuilder.makeCall( - SqlStdOperatorTable.IS_NOT_NULL, - fieldAccess); - } - return fieldAccess; - - case IN: - subQuery = subqueryMap.get(expr); - assert subQuery != null; - assert subQuery.expr != null : "expr != null"; - return subQuery.expr; - - case OVER: - return convertOver(this, expr); - - default: - // fall through - } - - // Apply standard conversions. - rex = expr.accept(this); - Util.permAssert(rex != null, "conversion result not null"); - return rex; - } - - /** - * Converts an item in an ORDER BY clause, extracting DESC, NULLS LAST - * and NULLS FIRST flags first. - */ - public RexNode convertSortExpression(SqlNode expr, Set flags) { - switch (expr.getKind()) { - case DESCENDING: - case NULLS_LAST: - case NULLS_FIRST: - flags.add(expr.getKind()); - final SqlNode operand = ((SqlCall) expr).operand(0); - return convertSortExpression(operand, flags); - default: - return convertExpression(expr); - } - } - - /** - * Determines whether a RexNode corresponds to a subquery that's been - * converted to a constant. - * - * @param rex the expression to be examined - * @return true if the expression is a dynamic parameter, a literal, or - * a literal that is being cast - */ - private boolean isConvertedSubq(RexNode rex) { - if ((rex instanceof RexLiteral) - || (rex instanceof RexDynamicParam)) { - return true; - } - if (rex instanceof RexCall) { - RexCall call = (RexCall) rex; - if (call.getOperator() == SqlStdOperatorTable.CAST) { - RexNode operand = call.getOperands().get(0); - if (operand instanceof RexLiteral) { - return true; - } - } - } - return false; - } - - // implement SqlRexContext - public int getGroupCount() { - if (agg != null) { - return agg.groupExprs.size(); - } - if (window != null) { - return window.isAlwaysNonEmpty() ? 1 : 0; - } - return -1; - } - - // implement SqlRexContext - public RexBuilder getRexBuilder() { - return rexBuilder; - } - - // implement SqlRexContext - public RexRangeRef getSubqueryExpr(SqlCall call) { - final SubQuery subQuery = subqueryMap.get(call); - assert subQuery != null; - return (RexRangeRef) subQuery.expr; - } - - // implement SqlRexContext - public RelDataTypeFactory getTypeFactory() { - return typeFactory; - } - - // implement SqlRexContext - public DefaultValueFactory getDefaultValueFactory() { - return defaultValueFactory; - } - - // implement SqlRexContext - public SqlValidator getValidator() { - return validator; - } - - // implement SqlRexContext - public RexNode convertLiteral(SqlLiteral literal) { - return exprConverter.convertLiteral(this, literal); - } - - public RexNode convertInterval(SqlIntervalQualifier intervalQualifier) { - return exprConverter.convertInterval(this, intervalQualifier); - } - - // implement SqlVisitor - public RexNode visit(SqlLiteral literal) { - return exprConverter.convertLiteral(this, literal); - } - - // implement SqlVisitor - public RexNode visit(SqlCall call) { - if (agg != null) { - final SqlOperator op = call.getOperator(); - if (op.isAggregator()) { - return agg.lookupAggregates(call); - } - } - return exprConverter.convertCall(this, call); - } - - // implement SqlVisitor - public RexNode visit(SqlNodeList nodeList) { - throw new UnsupportedOperationException(); - } - - // implement SqlVisitor - public RexNode visit(SqlIdentifier id) { - return convertIdentifier(this, id); - } - - // implement SqlVisitor - public RexNode visit(SqlDataTypeSpec type) { - throw new UnsupportedOperationException(); - } - - // implement SqlVisitor - public RexNode visit(SqlDynamicParam param) { - return convertDynamicParam(param); - } - - // implement SqlVisitor - public RexNode visit(SqlIntervalQualifier intervalQualifier) { - return convertInterval(intervalQualifier); - } - - public List getColumnMonotonicities() { - return columnMonotonicities; - } - } - - private static class DeferredLookup { - Blackboard bb; - String originalRelName; - - DeferredLookup( - Blackboard bb, - String originalRelName) { - this.bb = bb; - this.originalRelName = originalRelName; - } - - public RexFieldAccess getFieldAccess(String name) { - return (RexFieldAccess) bb.mapCorrelateVariableToRexNode.get(name); - } - - public String getOriginalRelName() { - return originalRelName; - } - } - - /** - * An implementation of DefaultValueFactory which always supplies NULL. - */ - class NullDefaultValueFactory implements DefaultValueFactory { - public boolean isGeneratedAlways( - RelOptTable table, - int iColumn) { - return false; - } - - public RexNode newColumnDefaultValue( - RelOptTable table, - int iColumn) { - return rexBuilder.constantNull(); - } - - public RexNode newAttributeInitializer( - RelDataType type, - SqlFunction constructor, - int iAttribute, - List constructorArgs) { - return rexBuilder.constantNull(); - } - } - - /** - * A default implementation of SubqueryConverter that does no conversion. - */ - private class NoOpSubqueryConverter implements SubqueryConverter { - // implement SubqueryConverter - public boolean canConvertSubquery() { - return false; - } - - // implement SubqueryConverter - public RexNode convertSubquery( - SqlCall subquery, - SqlToRelConverter parentConverter, - boolean isExists, - boolean isExplain) { - throw new IllegalArgumentException(); - } - } - - /** - * Converts expressions to aggregates. - * - *

Consider the expression SELECT deptno, SUM(2 * sal) FROM emp GROUP BY - * deptno Then - * - *

    - *
  • groupExprs = {SqlIdentifier(deptno)}
    • - *
  • convertedInputExprs = {RexInputRef(deptno), 2 * - * RefInputRef(sal)}
    • - *
  • inputRefs = {RefInputRef(#0), RexInputRef(#1)}
    • - *
  • aggCalls = {AggCall(SUM, {1})}
    • - *
- */ - protected class AggConverter implements SqlVisitor { - private final Blackboard bb; - - private final Map nameMap = - new HashMap(); - - /** - * The group-by expressions, in {@link SqlNode} format. - */ - private final SqlNodeList groupExprs = - new SqlNodeList(SqlParserPos.ZERO); - - /** - * Input expressions for the group columns and aggregates, in {@link - * RexNode} format. The first elements of the list correspond to the - * elements in {@link #groupExprs}; the remaining elements are for - * aggregates. - */ - private final List convertedInputExprs = - new ArrayList(); - - /** - * Names of {@link #convertedInputExprs}, where the expressions are - * simple mappings to input fields. - */ - private final List convertedInputExprNames = - new ArrayList(); - - private final List inputRefs = - new ArrayList(); - private final List aggCalls = - new ArrayList(); - private final Map aggMapping = - new HashMap(); - private final Map aggCallMapping = - new HashMap(); - - /** - * Creates an AggConverter. - * - *

The select parameter provides enough context to name - * aggregate calls which are top-level select list items. - * - * @param bb Blackboard - * @param select Query being translated; provides context to give - */ - public AggConverter(Blackboard bb, SqlSelect select) { - this.bb = bb; - - // Collect all expressions used in the select list so that aggregate - // calls can be named correctly. - final SqlNodeList selectList = select.getSelectList(); - for (int i = 0; i < selectList.size(); i++) { - SqlNode selectItem = selectList.get(i); - String name = null; - if (SqlUtil.isCallTo( - selectItem, - SqlStdOperatorTable.AS)) { - final SqlCall call = (SqlCall) selectItem; - selectItem = call.operand(0); - name = call.operand(1).toString(); - } - if (name == null) { - name = validator.deriveAlias(selectItem, i); - } - nameMap.put(selectItem.toString(), name); - } - } - - public void addGroupExpr(SqlNode expr) { - RexNode convExpr = bb.convertExpression(expr); - final RexNode rex = lookupGroupExpr(expr); - if (rex != null) { - return; // don't add duplicates, in e.g. "GROUP BY x, y, x" - } - groupExprs.add(expr); - String name = nameMap.get(expr.toString()); - addExpr(convExpr, name); - final RelDataType type = convExpr.getType(); - inputRefs.add(rexBuilder.makeInputRef(type, inputRefs.size())); - } - - /** - * Adds an expression, deducing an appropriate name if possible. - * - * @param expr Expression - * @param name Suggested name - */ - private void addExpr(RexNode expr, String name) { - convertedInputExprs.add(expr); - if ((name == null) && (expr instanceof RexInputRef)) { - final int i = ((RexInputRef) expr).getIndex(); - name = bb.root.getRowType().getFieldList().get(i).getName(); - } - if (convertedInputExprNames.contains(name)) { - // In case like 'SELECT ... GROUP BY x, y, x', don't add - // name 'x' twice. - name = null; - } - convertedInputExprNames.add(name); - } - - // implement SqlVisitor - public Void visit(SqlIdentifier id) { - return null; - } - - // implement SqlVisitor - public Void visit(SqlNodeList nodeList) { - for (int i = 0; i < nodeList.size(); i++) { - nodeList.get(i).accept(this); - } - return null; - } - - // implement SqlVisitor - public Void visit(SqlLiteral lit) { - return null; - } - - // implement SqlVisitor - public Void visit(SqlDataTypeSpec type) { - return null; - } - - // implement SqlVisitor - public Void visit(SqlDynamicParam param) { - return null; - } - - // implement SqlVisitor - public Void visit(SqlIntervalQualifier intervalQualifier) { - return null; - } - - public Void visit(SqlCall call) { - if (call.getOperator().isAggregator()) { - assert bb.agg == this; - List args = new ArrayList(); - List argTypes = - call.getOperator() instanceof SqlCountAggFunction - ? new ArrayList(call.getOperandList().size()) - : null; - try { - // switch out of agg mode - bb.agg = null; - for (SqlNode operand : call.getOperandList()) { - RexNode convertedExpr; - - // special case for COUNT(*): delete the * - if (operand instanceof SqlIdentifier) { - SqlIdentifier id = (SqlIdentifier) operand; - if (id.isStar()) { - assert call.operandCount() == 1; - assert args.isEmpty(); - break; - } - } - convertedExpr = bb.convertExpression(operand); - assert convertedExpr != null; - if (argTypes != null) { - argTypes.add(convertedExpr.getType()); - } - args.add(lookupOrCreateGroupExpr(convertedExpr)); - } - } finally { - // switch back into agg mode - bb.agg = this; - } - - final Aggregation aggregation = - (Aggregation) call.getOperator(); - RelDataType type = validator.deriveType(bb.scope, call); - boolean distinct = false; - SqlLiteral quantifier = call.getFunctionQuantifier(); - if ((null != quantifier) - && (quantifier.getValue() == SqlSelectKeyword.DISTINCT)) { - distinct = true; - } - final AggregateCall aggCall = - new AggregateCall( - aggregation, - distinct, - args, - type, - nameMap.get(call.toString())); - RexNode rex = - rexBuilder.addAggCall( - aggCall, - groupExprs.size(), - aggCalls, - aggCallMapping, - argTypes); - aggMapping.put(call, rex); - } else if (call instanceof SqlSelect) { - // rchen 2006-10-17: - // for now do not detect aggregates in subqueries. - return null; - } else { - for (SqlNode operand : call.getOperandList()) { - // Operands are occasionally null, e.g. switched CASE arg 0. - if (operand != null) { - operand.accept(this); - } - } - } - return null; - } - - private int lookupOrCreateGroupExpr(RexNode expr) { - for (int i = 0; i < convertedInputExprs.size(); i++) { - RexNode convertedInputExpr = convertedInputExprs.get(i); - if (expr.toString().equals(convertedInputExpr.toString())) { - return i; - } - } - - // not found -- add it - int index = convertedInputExprs.size(); - addExpr(expr, null); - return index; - } - - /** - * If an expression is structurally identical to one of the group-by - * expressions, returns a reference to the expression, otherwise returns - * null. - */ - public RexNode lookupGroupExpr(SqlNode expr) { - for (int i = 0; i < groupExprs.size(); i++) { - SqlNode groupExpr = groupExprs.get(i); - if (expr.equalsDeep(groupExpr, false)) { - return inputRefs.get(i); - } - } - return null; - } - - public RexNode lookupAggregates(SqlCall call) { - // assert call.getOperator().isAggregator(); - assert bb.agg == this; - - return aggMapping.get(call); - } - - public List getPreExprs() { - return convertedInputExprs; - } - - public List getPreNames() { - return convertedInputExprNames; - } - - public List getAggCalls() { - return aggCalls; - } - - public RelDataTypeFactory getTypeFactory() { - return typeFactory; - } - } - - /** - * Context to find a relational expression to a field offset. - */ - private static class LookupContext { - private final List> relOffsetList = - new ArrayList>(); - - /** - * Creates a LookupContext with multiple input relational expressions. - * - * @param bb Context for translating this subquery - * @param rels Relational expressions - * @param systemFieldCount Number of system fields - */ - LookupContext(Blackboard bb, List rels, int systemFieldCount) { - bb.flatten(rels, systemFieldCount, new int[]{0}, relOffsetList); - } - - /** - * Returns the relational expression with a given offset, and the - * ordinal in the combined row of its first field. - * - *

For example, in {@code Emp JOIN Dept}, findRel(1) returns the - * relational expression for {@code Dept} and offset 6 (because - * {@code Emp} has 6 fields, therefore the first field of {@code Dept} - * is field 6. - * - * @param offset Offset of relational expression in FROM clause - * @return Relational expression and the ordinal of its first field - */ - Pair findRel(int offset) { - return relOffsetList.get(offset); - } - } - - /** - * Shuttle which walks over a tree of {@link RexNode}s and applies 'over' to - * all agg functions. - * - *

This is necessary because the returned expression is not necessarily a - * call to an agg function. For example, - * - *

AVG(x)
- * - * becomes - * - *
SUM(x) / COUNT(x)
- * - *

Any aggregate functions are converted to calls to the internal - * $Histogram aggregation function and accessors such as - * $HistogramMin; for example, - * - *

MIN(x), MAX(x)
- * - * are converted to - * - *
$HistogramMin($Histogram(x)), - * $HistogramMax($Histogram(x))
- * - * Common sub-expression elmination will ensure that only one histogram is - * computed. - */ - private class HistogramShuttle extends RexShuttle { - /** - * Whether to convert calls to MIN(x) to HISTOGRAM_MIN(HISTOGRAM(x)). - * Histograms allow rolling computation, but require more space. - */ - static final boolean ENABLE_HISTOGRAM_AGG = false; - - private final List partitionKeys; - private final ImmutableList orderKeys; - private final RexWindowBound lowerBound; - private final RexWindowBound upperBound; - private final SqlWindow window; - - HistogramShuttle( - List partitionKeys, - ImmutableList orderKeys, - RexWindowBound lowerBound, RexWindowBound upperBound, - SqlWindow window) { - this.partitionKeys = partitionKeys; - this.orderKeys = orderKeys; - this.lowerBound = lowerBound; - this.upperBound = upperBound; - this.window = window; - } - - public RexNode visitCall(RexCall call) { - final SqlOperator op = call.getOperator(); - if (!(op instanceof SqlAggFunction)) { - return super.visitCall(call); - } - final SqlAggFunction aggOp = (SqlAggFunction) op; - final RelDataType type = call.getType(); - List exprs = call.getOperands(); - - SqlFunction histogramOp = !ENABLE_HISTOGRAM_AGG - ? null - : getHistogramOp(aggOp); - - if (histogramOp != null) { - final RelDataType histogramType = computeHistogramType(type); - - // For DECIMAL, since it's already represented as a bigint we - // want to do a reinterpretCast instead of a cast to avoid - // losing any precision. - boolean reinterpretCast = - type.getSqlTypeName() == SqlTypeName.DECIMAL; - - // Replace original expression with CAST of not one - // of the supported types - if (histogramType != type) { - exprs = new ArrayList(exprs); - exprs.set( - 0, - reinterpretCast - ? rexBuilder.makeReinterpretCast(histogramType, exprs.get(0), - rexBuilder.makeLiteral(false)) - : rexBuilder.makeCast(histogramType, exprs.get(0))); - } - - RexCallBinding bind = - new RexCallBinding( - rexBuilder.getTypeFactory(), - SqlStdOperatorTable.HISTOGRAM_AGG, - exprs); - - RexNode over = - rexBuilder.makeOver( - SqlStdOperatorTable.HISTOGRAM_AGG - .inferReturnType(bind), - SqlStdOperatorTable.HISTOGRAM_AGG, - exprs, - partitionKeys, - orderKeys, - lowerBound, - upperBound, - window.isRows(), - window.isAllowPartial(), - false); - - RexNode histogramCall = - rexBuilder.makeCall( - histogramType, - histogramOp, - ImmutableList.of(over)); - - // If needed, post Cast result back to original - // type. - if (histogramType != type) { - if (reinterpretCast) { - histogramCall = - rexBuilder.makeReinterpretCast( - type, - histogramCall, - rexBuilder.makeLiteral(false)); - } else { - histogramCall = - rexBuilder.makeCast(type, histogramCall); - } - } - - return histogramCall; - } else { - boolean needSum0 = aggOp == SqlStdOperatorTable.SUM - && type.isNullable(); - SqlAggFunction aggOpToUse = - needSum0 ? SqlStdOperatorTable.SUM0 - : aggOp; - return rexBuilder.makeOver( - type, - aggOpToUse, - exprs, - partitionKeys, - orderKeys, - lowerBound, - upperBound, - window.isRows(), - window.isAllowPartial(), - needSum0); - } - } - - /** - * Returns the histogram operator corresponding to a given aggregate - * function. - * - *

For example, getHistogramOp({@link - * SqlStdOperatorTable#MIN}} returns {@link - * SqlStdOperatorTable#HISTOGRAM_MIN}. - * - * @param aggFunction An aggregate function - * @return Its histogram function, or null - */ - SqlFunction getHistogramOp(SqlAggFunction aggFunction) { - if (aggFunction == SqlStdOperatorTable.MIN) { - return SqlStdOperatorTable.HISTOGRAM_MIN; - } else if (aggFunction == SqlStdOperatorTable.MAX) { - return SqlStdOperatorTable.HISTOGRAM_MAX; - } else if (aggFunction == SqlStdOperatorTable.FIRST_VALUE) { - return SqlStdOperatorTable.HISTOGRAM_FIRST_VALUE; - } else if (aggFunction == SqlStdOperatorTable.LAST_VALUE) { - return SqlStdOperatorTable.HISTOGRAM_LAST_VALUE; - } else { - return null; - } - } - - /** - * Returns the type for a histogram function. It is either the actual - * type or an an approximation to it. - */ - private RelDataType computeHistogramType(RelDataType type) { - if (SqlTypeUtil.isExactNumeric(type) - && type.getSqlTypeName() != SqlTypeName.BIGINT) { - return typeFactory.createSqlType(SqlTypeName.BIGINT); - } else if (SqlTypeUtil.isApproximateNumeric(type) - && type.getSqlTypeName() != SqlTypeName.DOUBLE) { - return typeFactory.createSqlType(SqlTypeName.DOUBLE); - } else { - return type; - } - } - } - - /** A sub-query, whether it needs to be translated using 2- or 3-valued - * logic. */ - private static class SubQuery { - final SqlNode node; - final RelOptUtil.Logic logic; - RexNode expr; - - private SubQuery(SqlNode node, RelOptUtil.Logic logic) { - this.node = node; - this.logic = logic; - } - } -} - -// End SqlToRelConverter.java From 5eecadb1d798624c4e797a7eaadff5736b51f0c6 Mon Sep 17 00:00:00 2001 From: "qianhao.zhou" Date: Mon, 1 Dec 2014 16:17:53 +0800 Subject: [PATCH 14/35] fix sync error on tables with same name but different db issue: https://github.com/KylinOLAP/Kylin/issues/84 --- .../kylinolap/metadata/tool/HiveSourceTableLoader.java | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/metadata/src/main/java/com/kylinolap/metadata/tool/HiveSourceTableLoader.java b/metadata/src/main/java/com/kylinolap/metadata/tool/HiveSourceTableLoader.java index 040ba8d..3e4cca3 100644 --- a/metadata/src/main/java/com/kylinolap/metadata/tool/HiveSourceTableLoader.java +++ b/metadata/src/main/java/com/kylinolap/metadata/tool/HiveSourceTableLoader.java @@ -28,6 +28,7 @@ import java.util.Set; import java.util.UUID; +import com.kylinolap.metadata.MetadataManager; import org.apache.commons.io.IOUtils; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; @@ -75,6 +76,20 @@ metaTmpDir.delete(); metaTmpDir.mkdirs(); + for (String database: db2tables.keySet()) { + for (String table: db2tables.get(database)) { + TableDesc tableDesc = MetadataManager.getInstance(config).getTableDesc(table); + if (tableDesc == null) { + continue; + } + if (tableDesc.getDatabase().equalsIgnoreCase(database)) { + continue; + } else { + throw new UnsupportedOperationException(String.format("there is already a table[%s] in database[%s]", tableDesc.getName(), tableDesc.getDatabase())); + } + } + } + // extract from hive Set loadedTables = Sets.newHashSet(); for (String database : db2tables.keySet()) { From 56c7970cfb8629cdafb8bd7501e61182d73994db Mon Sep 17 00:00:00 2001 From: yangli9 Date: Mon, 1 Dec 2014 22:12:33 +0800 Subject: [PATCH 15/35] fix compile warnings --- common/src/test/java/com/kylinolap/common/util/MailServiceTest.java | 1 + cube/src/main/java/com/kylinolap/cube/CubeInstance.java | 1 - job/src/test/java/com/kylinolap/job/BuildCubeWithEngineTest.java | 3 +-- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/common/src/test/java/com/kylinolap/common/util/MailServiceTest.java b/common/src/test/java/com/kylinolap/common/util/MailServiceTest.java index 33a51e8..044d134 100644 --- a/common/src/test/java/com/kylinolap/common/util/MailServiceTest.java +++ b/common/src/test/java/com/kylinolap/common/util/MailServiceTest.java @@ -30,6 +30,7 @@ @Test public void testSendEmail() throws IOException { + @SuppressWarnings("deprecation") KylinConfig config = KylinConfig.getInstanceForTest(AbstractKylinTestCase.SANDBOX_TEST_DATA); MailService mailservice = new MailService(config); diff --git a/cube/src/main/java/com/kylinolap/cube/CubeInstance.java b/cube/src/main/java/com/kylinolap/cube/CubeInstance.java index 4294934..942332c 100644 --- a/cube/src/main/java/com/kylinolap/cube/CubeInstance.java +++ b/cube/src/main/java/com/kylinolap/cube/CubeInstance.java @@ -25,7 +25,6 @@ import com.fasterxml.jackson.annotation.JsonManagedReference; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Objects; -import com.google.common.base.Preconditions; import com.google.common.collect.Lists; import com.kylinolap.common.KylinConfig; import com.kylinolap.common.persistence.ResourceStore; diff --git a/job/src/test/java/com/kylinolap/job/BuildCubeWithEngineTest.java b/job/src/test/java/com/kylinolap/job/BuildCubeWithEngineTest.java index 4af5932..c15f463 100644 --- a/job/src/test/java/com/kylinolap/job/BuildCubeWithEngineTest.java +++ b/job/src/test/java/com/kylinolap/job/BuildCubeWithEngineTest.java @@ -24,15 +24,14 @@ import java.util.ArrayList; import java.util.List; import java.util.TimeZone; -import java.util.UUID; -import com.google.common.collect.Lists; import org.junit.After; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; import org.quartz.SchedulerException; +import com.google.common.collect.Lists; import com.kylinolap.common.KylinConfig; import com.kylinolap.common.util.ClasspathUtil; import com.kylinolap.common.util.HBaseMetadataTestCase; From 3d8df396d8ec547de9b1920d3c89ed1c322245e3 Mon Sep 17 00:00:00 2001 From: Akash Mishra Date: Tue, 2 Dec 2014 19:09:12 +0530 Subject: [PATCH 16/35] Fixing the error message as null is being replaced in place of JAVA_HOME --- deploy.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy.sh b/deploy.sh index f3bad01..82613a3 100755 --- a/deploy.sh +++ b/deploy.sh @@ -38,7 +38,7 @@ echo "Checking JAVA status..." if [ -z "$JAVA_HOME" ] then - echo "Please set $JAVA_HOME so that Kylin-Deploy can proceed" + echo "Please set JAVA_HOME so that Kylin-Deploy can proceed" exit 1 else echo "JAVA_HOME is set to $JAVA_HOME" From fe516acdf116ac7d36e5b32a05c8cd2bffda9694 Mon Sep 17 00:00:00 2001 From: yangli9 Date: Tue, 2 Dec 2014 11:49:41 -0800 Subject: [PATCH 17/35] fix error handling --- .../kylinolap/rest/controller/CubeController.java | 8 ++-- .../com/kylinolap/rest/service/CubeService.java | 50 ++++++++++------------ 2 files changed, 26 insertions(+), 32 deletions(-) diff --git a/server/src/main/java/com/kylinolap/rest/controller/CubeController.java b/server/src/main/java/com/kylinolap/rest/controller/CubeController.java index 7a36d77..0822231 100644 --- a/server/src/main/java/com/kylinolap/rest/controller/CubeController.java +++ b/server/src/main/java/com/kylinolap/rest/controller/CubeController.java @@ -38,6 +38,7 @@ import com.codahale.metrics.annotation.Metered; import com.fasterxml.jackson.core.JsonParseException; +import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonMappingException; import com.kylinolap.common.util.JsonUtil; import com.kylinolap.cube.CubeBuildTypeEnum; @@ -327,12 +328,13 @@ public CubeRequest saveCubeDesc(@RequestBody CubeRequest cubeRequest) { * Get available table list of the input database * * @return Table metadata array + * @throws JsonProcessingException * @throws IOException */ @RequestMapping(value = "", method = { RequestMethod.PUT }) @ResponseBody @Metered(name = "updateCube") - public CubeRequest updateCubeDesc(@RequestBody CubeRequest cubeRequest) { + public CubeRequest updateCubeDesc(@RequestBody CubeRequest cubeRequest) throws JsonProcessingException { CubeDesc desc = deserializeCubeDesc(cubeRequest); if (desc == null) { @@ -346,18 +348,15 @@ public CubeRequest updateCubeDesc(@RequestBody CubeRequest cubeRequest) { return cubeRequest; } - String descData = ""; try { CubeInstance cube = cubeService.getCubeManager().getCube(cubeRequest.getCubeName()); String projectName = (null == cubeRequest.getProject()) ? ProjectInstance.DEFAULT_PROJECT_NAME : cubeRequest.getProject(); desc = cubeService.updateCubeAndDesc(cube, desc, projectName); - descData = JsonUtil.writeValueAsIndentString(desc); } catch (AccessDeniedException accessDeniedException) { throw new ForbiddenException("You don't have right to update this cube."); } catch (Exception e) { logger.error("Failed to deal with the request.", e); - e.printStackTrace(); throw new InternalErrorException("Failed to deal with the request.", e); } @@ -366,6 +365,7 @@ public CubeRequest updateCubeDesc(@RequestBody CubeRequest cubeRequest) { } else { updateRequest(cubeRequest, false, omitMessage(desc.getError())); } + String descData = JsonUtil.writeValueAsIndentString(desc); cubeRequest.setCubeDescData(descData); return cubeRequest; diff --git a/server/src/main/java/com/kylinolap/rest/service/CubeService.java b/server/src/main/java/com/kylinolap/rest/service/CubeService.java index e1f56c1..0dd040b 100644 --- a/server/src/main/java/com/kylinolap/rest/service/CubeService.java +++ b/server/src/main/java/com/kylinolap/rest/service/CubeService.java @@ -169,15 +169,15 @@ public CubeInstance createCubeAndDesc(String cubeName, String projectName, CubeD getMetadataManager().removeCubeDesc(createdDesc); throw new InternalErrorException(createdDesc.getError().get(0)); } - - try{ + + try { int cuboidCount = CuboidCLI.simulateCuboidGeneration(createdDesc); logger.info("New cube " + cubeName + " has " + cuboidCount + " cuboids"); - }catch(Exception e){ + } catch (Exception e) { getMetadataManager().removeCubeDesc(createdDesc); throw new InternalErrorException("Failed to deal with the request.", e); } - + createdCube = getCubeManager().createCube(cubeName, projectName, createdDesc, owner); accessService.init(createdCube, AclPermission.ADMINISTRATION); @@ -188,7 +188,7 @@ public CubeInstance createCubeAndDesc(String cubeName, String projectName, CubeD } @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN + " or hasPermission(#cube, 'ADMINISTRATION') or hasPermission(#cube, 'MANAGEMENT')") - public CubeDesc updateCubeAndDesc(CubeInstance cube, CubeDesc desc, String newProjectName) throws UnknownHostException, IOException, JobException { + public CubeDesc updateCubeAndDesc(CubeInstance cube, CubeDesc desc, String newProjectName) throws Exception { List jobInstances = this.getJobManager().listJobs(cube.getName(), null); for (JobInstance jobInstance : jobInstances) { if (jobInstance.getStatus() == JobStatusEnum.PENDING || jobInstance.getStatus() == JobStatusEnum.RUNNING) { @@ -196,28 +196,22 @@ public CubeDesc updateCubeAndDesc(CubeInstance cube, CubeDesc desc, String newPr } } - try { - if (!cube.getDescriptor().calculateSignature().equals(cube.getDescriptor().getSignature())) { - this.releaseAllSegments(cube); - } - - CubeDesc updatedCubeDesc = getMetadataManager().updateCubeDesc(desc); - - int cuboidCount = CuboidCLI.simulateCuboidGeneration(updatedCubeDesc); - logger.info("Updated cube " + cube.getName() + " has " + cuboidCount + " cuboids"); - - if (!getProjectManager().isCubeInProject(newProjectName, cube)) { - String owner = SecurityContextHolder.getContext().getAuthentication().getName(); - ProjectInstance newProject = getProjectManager().updateCubeToProject(cube.getName(), newProjectName, owner); - accessService.inherit(cube, newProject); - } + if (!cube.getDescriptor().calculateSignature().equals(cube.getDescriptor().getSignature())) { + this.releaseAllSegments(cube); + } - return updatedCubeDesc; - } catch (IOException e) { - throw new InternalErrorException("Failed to deal with the request.", e); - } catch (CubeIntegrityException e) { - throw new InternalErrorException("Failed to deal with the request.", e); + CubeDesc updatedCubeDesc = getMetadataManager().updateCubeDesc(desc); + + int cuboidCount = CuboidCLI.simulateCuboidGeneration(updatedCubeDesc); + logger.info("Updated cube " + cube.getName() + " has " + cuboidCount + " cuboids"); + + if (!getProjectManager().isCubeInProject(newProjectName, cube)) { + String owner = SecurityContextHolder.getContext().getAuthentication().getName(); + ProjectInstance newProject = getProjectManager().updateCubeToProject(cube.getName(), newProjectName, owner); + accessService.inherit(cube, newProject); } + + return updatedCubeDesc; } @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN + " or hasPermission(#cube, 'ADMINISTRATION') or hasPermission(#cube, 'MANAGEMENT')") @@ -594,9 +588,9 @@ private void releaseAllSegments(CubeInstance cube) throws IOException, JobExcept getMetadataManager().reload(); return (String[]) loaded.toArray(new String[loaded.size()]); } - + @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN) - public void syncTableToProject(String tables,String project) throws IOException { + public void syncTableToProject(String tables, String project) throws IOException { getProjectManager().updateTableToProject(tables, project); - } + } } From 7597accc6626b6c57efe5e4f0cce07b8dcf10efe Mon Sep 17 00:00:00 2001 From: yangli9 Date: Tue, 2 Dec 2014 12:01:41 -0800 Subject: [PATCH 18/35] fix error handling on cube update --- server/src/main/java/com/kylinolap/rest/service/CubeService.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/server/src/main/java/com/kylinolap/rest/service/CubeService.java b/server/src/main/java/com/kylinolap/rest/service/CubeService.java index 0dd040b..8534b52 100644 --- a/server/src/main/java/com/kylinolap/rest/service/CubeService.java +++ b/server/src/main/java/com/kylinolap/rest/service/CubeService.java @@ -201,6 +201,8 @@ public CubeDesc updateCubeAndDesc(CubeInstance cube, CubeDesc desc, String newPr } CubeDesc updatedCubeDesc = getMetadataManager().updateCubeDesc(desc); + if (updatedCubeDesc.getError().size() > 0) + return updatedCubeDesc; int cuboidCount = CuboidCLI.simulateCuboidGeneration(updatedCubeDesc); logger.info("Updated cube " + cube.getName() + " has " + cuboidCount + " cuboids"); From d9f25852556c836c90278dc1a6e8fc4a03af6176 Mon Sep 17 00:00:00 2001 From: yangli9 Date: Tue, 2 Dec 2014 13:24:21 -0800 Subject: [PATCH 19/35] fix error handling --- server/src/main/java/com/kylinolap/rest/controller/CubeController.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/com/kylinolap/rest/controller/CubeController.java b/server/src/main/java/com/kylinolap/rest/controller/CubeController.java index 0822231..74cfd9b 100644 --- a/server/src/main/java/com/kylinolap/rest/controller/CubeController.java +++ b/server/src/main/java/com/kylinolap/rest/controller/CubeController.java @@ -356,13 +356,13 @@ public CubeRequest updateCubeDesc(@RequestBody CubeRequest cubeRequest) throws J } catch (AccessDeniedException accessDeniedException) { throw new ForbiddenException("You don't have right to update this cube."); } catch (Exception e) { - logger.error("Failed to deal with the request.", e); throw new InternalErrorException("Failed to deal with the request.", e); } if (desc.getError().isEmpty()) { cubeRequest.setSuccessful(true); } else { + logger.warn("Cube " + desc.getName() + " fail to create because " + desc.getError()); updateRequest(cubeRequest, false, omitMessage(desc.getError())); } String descData = JsonUtil.writeValueAsIndentString(desc); From 5872c49850a58ca8a9bfcefa5aa6aabca967a710 Mon Sep 17 00:00:00 2001 From: yangli9 Date: Tue, 2 Dec 2014 14:05:35 -0800 Subject: [PATCH 20/35] tolerate "" for date field --- .../main/java/com/kylinolap/dict/DateStrDictionary.java | 5 +++++ .../src/main/java/com/kylinolap/dict/Dictionary.java | 14 +++++++++++--- .../main/java/com/kylinolap/dict/DictionaryGenerator.java | 3 +++ 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/dictionary/src/main/java/com/kylinolap/dict/DateStrDictionary.java b/dictionary/src/main/java/com/kylinolap/dict/DateStrDictionary.java index 39af590..ed26aae 100644 --- a/dictionary/src/main/java/com/kylinolap/dict/DateStrDictionary.java +++ b/dictionary/src/main/java/com/kylinolap/dict/DateStrDictionary.java @@ -138,6 +138,11 @@ public int getSizeOfValue() { } @Override + protected boolean isNullByteForm(byte[] value, int offset, int len) { + return value == null || len == 0; + } + + @Override final protected int getIdFromValueImpl(String value, int roundFlag) { Date date = stringToDate(value, pattern); int id = calcIdFromSeqNo(getNumOfDaysSince0000(date)); diff --git a/dictionary/src/main/java/com/kylinolap/dict/Dictionary.java b/dictionary/src/main/java/com/kylinolap/dict/Dictionary.java index f0ccbb3..b54c6a3 100644 --- a/dictionary/src/main/java/com/kylinolap/dict/Dictionary.java +++ b/dictionary/src/main/java/com/kylinolap/dict/Dictionary.java @@ -59,7 +59,7 @@ * value of column */ abstract public int getSizeOfValue(); - + /** * Convenient form of getIdFromValue(value, 0) */ @@ -79,12 +79,16 @@ final public int getIdFromValue(T value) { * failed */ final public int getIdFromValue(T value, int roundingFlag) { - if (value == null) + if (isNullObjectForm(value)) return nullId(); else return getIdFromValueImpl(value, roundingFlag); } + protected boolean isNullObjectForm(T value) { + return value == null; + } + abstract protected int getIdFromValueImpl(T value, int roundingFlag); /** @@ -122,11 +126,15 @@ final public int getIdFromValueBytes(byte[] value, int offset, int len) { * failed */ final public int getIdFromValueBytes(byte[] value, int offset, int len, int roundingFlag) { - if (value == null) + if (isNullByteForm(value, offset, len)) return nullId(); else return getIdFromValueBytesImpl(value, offset, len, roundingFlag); } + + protected boolean isNullByteForm(byte[] value, int offset, int len) { + return value == null; + } abstract protected int getIdFromValueBytesImpl(byte[] value, int offset, int len, int roundingFlag); diff --git a/dictionary/src/main/java/com/kylinolap/dict/DictionaryGenerator.java b/dictionary/src/main/java/com/kylinolap/dict/DictionaryGenerator.java index e1b75fe..13e36ed 100644 --- a/dictionary/src/main/java/com/kylinolap/dict/DictionaryGenerator.java +++ b/dictionary/src/main/java/com/kylinolap/dict/DictionaryGenerator.java @@ -116,6 +116,9 @@ private static Dictionary buildDateStrDict(List values, int baseId, int matchPattern = ptn; // be optimistic SimpleDateFormat sdf = new SimpleDateFormat(ptn); for (byte[] value : values) { + if (value.length == 0) + continue; + String str = Bytes.toString(value); try { sdf.parse(str); From 929937fbd23eeb781e921a1283aec3273fbeae62 Mon Sep 17 00:00:00 2001 From: yangli9 Date: Tue, 2 Dec 2014 15:58:44 -0800 Subject: [PATCH 21/35] more log for unrecognized date string --- dictionary/src/main/java/com/kylinolap/dict/DictionaryGenerator.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dictionary/src/main/java/com/kylinolap/dict/DictionaryGenerator.java b/dictionary/src/main/java/com/kylinolap/dict/DictionaryGenerator.java index 13e36ed..213fd46 100644 --- a/dictionary/src/main/java/com/kylinolap/dict/DictionaryGenerator.java +++ b/dictionary/src/main/java/com/kylinolap/dict/DictionaryGenerator.java @@ -126,6 +126,7 @@ private static Dictionary buildDateStrDict(List values, int baseId, int samples.add(str); } catch (ParseException e) { // not match pattern, try next + logger.info("Unrecognized datetime value: " + str); matchPattern = null; break; } @@ -133,7 +134,7 @@ private static Dictionary buildDateStrDict(List values, int baseId, int if (matchPattern != null) return new DateStrDictionary(matchPattern, baseId); } - throw new IllegalStateException("Unrecognized datetime values: " + samples); + throw new IllegalStateException("Unrecognized datetime value"); } private static Dictionary buildStringDict(List values, int baseId, int nSamples, ArrayList samples) { From 0835c99da5bd277fc226d6e2cc1e32f16239020b Mon Sep 17 00:00:00 2001 From: yangli9 Date: Tue, 2 Dec 2014 16:09:34 -0800 Subject: [PATCH 22/35] be tolerable for a few unmatching date values --- .../main/java/com/kylinolap/dict/DictionaryGenerator.java | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/dictionary/src/main/java/com/kylinolap/dict/DictionaryGenerator.java b/dictionary/src/main/java/com/kylinolap/dict/DictionaryGenerator.java index 213fd46..d367782 100644 --- a/dictionary/src/main/java/com/kylinolap/dict/DictionaryGenerator.java +++ b/dictionary/src/main/java/com/kylinolap/dict/DictionaryGenerator.java @@ -111,24 +111,29 @@ public static Dictionary mergeDictionaries(DictionaryInfo targetInfo, List values, int baseId, int nSamples, ArrayList samples) { + final int BAD_THRESHOLD = 2; String matchPattern = null; + for (String ptn : DATE_PATTERNS) { matchPattern = ptn; // be optimistic + int badCount = 0; SimpleDateFormat sdf = new SimpleDateFormat(ptn); for (byte[] value : values) { if (value.length == 0) continue; - + String str = Bytes.toString(value); try { sdf.parse(str); if (samples.size() < nSamples && samples.contains(str) == false) samples.add(str); } catch (ParseException e) { - // not match pattern, try next logger.info("Unrecognized datetime value: " + str); - matchPattern = null; - break; + badCount++; + if (badCount >= BAD_THRESHOLD) { + matchPattern = null; + break; + } } } if (matchPattern != null) From d2cab625322882afc73d1ba5829f551fdb058493 Mon Sep 17 00:00:00 2001 From: Kejia-Wang Date: Wed, 3 Dec 2014 12:46:37 +0800 Subject: [PATCH 23/35] Job status multiple checkbox --- webapp/app/js/controllers/job.js | 32 ++++++++++---- webapp/app/partials/jobs/job_steps.html | 6 +-- webapp/app/partials/jobs/jobs.html | 77 +++++++++++++++++++-------------- 3 files changed, 72 insertions(+), 43 deletions(-) diff --git a/webapp/app/js/controllers/job.js b/webapp/app/js/controllers/job.js index 68b1f37..7c768dc 100644 --- a/webapp/app/js/controllers/job.js +++ b/webapp/app/js/controllers/job.js @@ -6,13 +6,13 @@ KylinApp $scope.jobs = {}; $scope.projects = []; $scope.action = {}; - $scope.allStatus = [ - {name: 'new', value: 0}, - {name: 'pending', value: 1}, - {name: 'running', value: 2}, - {name: 'finished', value: 4}, - {name: 'error', value: 8}, - {name: 'discarded', value: 16} + $scope.allStatus = [ //[$scope.allStatus[0], $scope.allStatus[1], $scope.allStatus[2], $scope.allStatus[3], $scope.allStatus[4], $scope.allStatus[5]] + {name: 'NEW', value: 0}, + {name: 'PENDING', value: 1}, + {name: 'RUNNING', value: 2}, + {name: 'FINISHED', value: 4}, + {name: 'ERROR', value: 8}, + {name: 'DISCARDED', value: 16} ]; $scope.theaditems = [ {attr: 'name', name: 'Job Name'}, @@ -21,7 +21,23 @@ KylinApp {attr: 'last_modified', name: 'Last Modified Time'}, {attr: 'duration', name: 'Duration'} ]; - $scope.status = [$scope.allStatus[0], $scope.allStatus[1], $scope.allStatus[2], $scope.allStatus[3], $scope.allStatus[4], $scope.allStatus[5]]; + $scope.status = []; + $scope.isAll = null; + $scope.toggleSelection = function toggleSelection(current) { + if(current == 'ALL'){ + $scope.status = $scope.isAll ? [] : $scope.allStatus.slice(0); + }else{ + var idx = $scope.status.indexOf(current); + if (idx > -1) { + $scope.status.splice(idx, 1); + }else { + $scope.status.push(current); + } + } + $scope.jobs={}; + $scope.reload(); + }; + // projectName from page ctrl $scope.state = {loading: false, refreshing: false, filterAttr: 'last_modified', filterReverse: true, reverseColumn: 'last_modified', projectName:$scope.project.selectedProject}; diff --git a/webapp/app/partials/jobs/job_steps.html b/webapp/app/partials/jobs/job_steps.html index 3d1b521..bd7320d 100644 --- a/webapp/app/partials/jobs/job_steps.html +++ b/webapp/app/partials/jobs/job_steps.html @@ -29,10 +29,10 @@ {{state.selectedJob.job_status}} @@ -70,7 +70,7 @@ {{step.info.mr_job_id ? ('MR Job:
' + step.info.mr_job_id + '
') :''}}" ng-class="{ 'fa fa-ellipsis-h bg-gray' : step.step_status=='PENDING', - 'fa fa-spinner fa-spin btn-yellow' : step.step_status=='WAITING' || step.step_status=='RUNNING', + 'fa fa-spinner fa-spin bg-aqua' : step.step_status=='WAITING' || step.step_status=='RUNNING', 'fa fa-check bg-green' : step.step_status=='FINISHED', 'fa fa-warning bg-red' : step.step_status=='ERROR', 'fa fa-ban bg-navy' : step.step_status=='DISCARDED' diff --git a/webapp/app/partials/jobs/jobs.html b/webapp/app/partials/jobs/jobs.html index faf02e0..95dd57d 100644 --- a/webapp/app/partials/jobs/jobs.html +++ b/webapp/app/partials/jobs/jobs.html @@ -1,43 +1,56 @@ -

- -
-
- Project: - - - - - - -
-
- - +
+ +
+
+
+ Project: + + + + + + +
+
+
+ +
Cube Name:
- -
- - - Job Status: - - +
+ + + Job Status: + + + + -
+
From 6da8d4d18b7ce65a28660bc2dae2c5e74ee420b8 Mon Sep 17 00:00:00 2001 From: Kejia-Wang Date: Wed, 3 Dec 2014 16:28:14 +0800 Subject: [PATCH 24/35] Move job status filter to jobs table header --- webapp/app/js/controllers/job.js | 19 ++++------- webapp/app/partials/cubes/cubes.html | 40 ++++++++++++----------- webapp/app/partials/jobs/jobs.html | 63 ++++++++++++++++-------------------- 3 files changed, 55 insertions(+), 67 deletions(-) diff --git a/webapp/app/js/controllers/job.js b/webapp/app/js/controllers/job.js index 7c768dc..e72e33f 100644 --- a/webapp/app/js/controllers/job.js +++ b/webapp/app/js/controllers/job.js @@ -6,7 +6,7 @@ KylinApp $scope.jobs = {}; $scope.projects = []; $scope.action = {}; - $scope.allStatus = [ //[$scope.allStatus[0], $scope.allStatus[1], $scope.allStatus[2], $scope.allStatus[3], $scope.allStatus[4], $scope.allStatus[5]] + $scope.allStatus = [ {name: 'NEW', value: 0}, {name: 'PENDING', value: 1}, {name: 'RUNNING', value: 2}, @@ -22,20 +22,13 @@ KylinApp {attr: 'duration', name: 'Duration'} ]; $scope.status = []; - $scope.isAll = null; $scope.toggleSelection = function toggleSelection(current) { - if(current == 'ALL'){ - $scope.status = $scope.isAll ? [] : $scope.allStatus.slice(0); - }else{ - var idx = $scope.status.indexOf(current); - if (idx > -1) { - $scope.status.splice(idx, 1); - }else { - $scope.status.push(current); - } + var idx = $scope.status.indexOf(current); + if (idx > -1) { + $scope.status.splice(idx, 1); + }else { + $scope.status.push(current); } - $scope.jobs={}; - $scope.reload(); }; // projectName from page ctrl diff --git a/webapp/app/partials/cubes/cubes.html b/webapp/app/partials/cubes/cubes.html index aff17d5..1e973e1 100644 --- a/webapp/app/partials/cubes/cubes.html +++ b/webapp/app/partials/cubes/cubes.html @@ -1,30 +1,32 @@ -
+
-
-
- Project: - - - - - - -
-
+
+
+
+ Project: + + + + + + +
+
+
- +
Cube Name:
- +
Json Editor Cube diff --git a/webapp/app/partials/jobs/jobs.html b/webapp/app/partials/jobs/jobs.html index 95dd57d..d3b6416 100644 --- a/webapp/app/partials/jobs/jobs.html +++ b/webapp/app/partials/jobs/jobs.html @@ -12,7 +12,7 @@ - +
@@ -26,47 +26,40 @@ -
- - - Job Status: - - - - - - - +
-
-
-
-
- -
-
+
+
-
+
+
+ +
+ + + +
+
+
+ +
+
+
+ +
+
- + +
Date: Wed, 3 Dec 2014 15:11:29 -0800 Subject: [PATCH 25/35] minor change in DateStrDict --- dictionary/src/main/java/com/kylinolap/dict/DictionaryGenerator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dictionary/src/main/java/com/kylinolap/dict/DictionaryGenerator.java b/dictionary/src/main/java/com/kylinolap/dict/DictionaryGenerator.java index d367782..16ae61c 100644 --- a/dictionary/src/main/java/com/kylinolap/dict/DictionaryGenerator.java +++ b/dictionary/src/main/java/com/kylinolap/dict/DictionaryGenerator.java @@ -130,7 +130,7 @@ private static Dictionary buildDateStrDict(List values, int baseId, int } catch (ParseException e) { logger.info("Unrecognized datetime value: " + str); badCount++; - if (badCount >= BAD_THRESHOLD) { + if (badCount > BAD_THRESHOLD) { matchPattern = null; break; } From 6f4588eff50b0b5633b42aecdf3b59341d4fc79d Mon Sep 17 00:00:00 2001 From: Luke Han Date: Wed, 3 Dec 2014 16:06:17 -0800 Subject: [PATCH 26/35] [maven-release-plugin] prepare release kylin-0.6.3 --- atopcalcite/pom.xml | 2 +- common/pom.xml | 2 +- cube/pom.xml | 2 +- dictionary/pom.xml | 2 +- jdbc/pom.xml | 2 +- job/pom.xml | 2 +- metadata/pom.xml | 2 +- pom.xml | 4 ++-- query/pom.xml | 2 +- server/pom.xml | 2 +- storage/pom.xml | 2 +- 11 files changed, 12 insertions(+), 12 deletions(-) diff --git a/atopcalcite/pom.xml b/atopcalcite/pom.xml index e36f434..aab0f7f 100644 --- a/atopcalcite/pom.xml +++ b/atopcalcite/pom.xml @@ -8,7 +8,7 @@ com.kylinolap kylin - 0.6.3-SNAPSHOT + 0.6.3 diff --git a/common/pom.xml b/common/pom.xml index 32a46c6..53c7f4e 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -8,7 +8,7 @@ com.kylinolap kylin - 0.6.3-SNAPSHOT + 0.6.3 diff --git a/cube/pom.xml b/cube/pom.xml index f177c07..490a817 100644 --- a/cube/pom.xml +++ b/cube/pom.xml @@ -7,7 +7,7 @@ com.kylinolap kylin - 0.6.3-SNAPSHOT + 0.6.3 diff --git a/dictionary/pom.xml b/dictionary/pom.xml index 1d09903..be1ced3 100644 --- a/dictionary/pom.xml +++ b/dictionary/pom.xml @@ -8,7 +8,7 @@ com.kylinolap kylin - 0.6.3-SNAPSHOT + 0.6.3 diff --git a/jdbc/pom.xml b/jdbc/pom.xml index e69cac0..4fc37a7 100644 --- a/jdbc/pom.xml +++ b/jdbc/pom.xml @@ -11,7 +11,7 @@ com.kylinolap kylin - 0.6.3-SNAPSHOT + 0.6.3 diff --git a/job/pom.xml b/job/pom.xml index b039957..05cb6ad 100644 --- a/job/pom.xml +++ b/job/pom.xml @@ -4,7 +4,7 @@ com.kylinolap kylin - 0.6.3-SNAPSHOT + 0.6.3 kylin-job diff --git a/metadata/pom.xml b/metadata/pom.xml index 3e76619..a77c88a 100644 --- a/metadata/pom.xml +++ b/metadata/pom.xml @@ -8,7 +8,7 @@ com.kylinolap kylin - 0.6.3-SNAPSHOT + 0.6.3 diff --git a/pom.xml b/pom.xml index c38a8cc..1e1f606 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ com.kylinolap kylin pom - 0.6.3-SNAPSHOT + 0.6.3 Kylin:HadoopOLAPEngine @@ -493,7 +493,7 @@ scm:git:git@github.com:KylinOLAP/Kylin.git scm:git:git@github.com:KylinOLAP/Kylin.git scm:git:git@github.com:KylinOLAP/Kylin.git - HEAD + kylin-0.6.3 diff --git a/query/pom.xml b/query/pom.xml index 0f035b3..e0c7f38 100644 --- a/query/pom.xml +++ b/query/pom.xml @@ -9,7 +9,7 @@ com.kylinolap kylin - 0.6.3-SNAPSHOT + 0.6.3 diff --git a/server/pom.xml b/server/pom.xml index fea3397..9315dfe 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -9,7 +9,7 @@ com.kylinolap kylin - 0.6.3-SNAPSHOT + 0.6.3 diff --git a/storage/pom.xml b/storage/pom.xml index 46c43e5..520fd0e 100644 --- a/storage/pom.xml +++ b/storage/pom.xml @@ -7,7 +7,7 @@ com.kylinolap kylin - 0.6.3-SNAPSHOT + 0.6.3 From 3f5608e523330230f30608b60e8cd1bac6d82076 Mon Sep 17 00:00:00 2001 From: Luke Han Date: Wed, 3 Dec 2014 16:06:20 -0800 Subject: [PATCH 27/35] [maven-release-plugin] prepare for next development iteration --- atopcalcite/pom.xml | 2 +- common/pom.xml | 2 +- cube/pom.xml | 2 +- dictionary/pom.xml | 2 +- jdbc/pom.xml | 2 +- job/pom.xml | 2 +- metadata/pom.xml | 2 +- pom.xml | 4 ++-- query/pom.xml | 2 +- server/pom.xml | 2 +- storage/pom.xml | 2 +- 11 files changed, 12 insertions(+), 12 deletions(-) diff --git a/atopcalcite/pom.xml b/atopcalcite/pom.xml index aab0f7f..427927f 100644 --- a/atopcalcite/pom.xml +++ b/atopcalcite/pom.xml @@ -8,7 +8,7 @@ com.kylinolap kylin - 0.6.3 + 0.6.4-SNAPSHOT diff --git a/common/pom.xml b/common/pom.xml index 53c7f4e..d7f729d 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -8,7 +8,7 @@ com.kylinolap kylin - 0.6.3 + 0.6.4-SNAPSHOT diff --git a/cube/pom.xml b/cube/pom.xml index 490a817..6004999 100644 --- a/cube/pom.xml +++ b/cube/pom.xml @@ -7,7 +7,7 @@ com.kylinolap kylin - 0.6.3 + 0.6.4-SNAPSHOT diff --git a/dictionary/pom.xml b/dictionary/pom.xml index be1ced3..2e00b70 100644 --- a/dictionary/pom.xml +++ b/dictionary/pom.xml @@ -8,7 +8,7 @@ com.kylinolap kylin - 0.6.3 + 0.6.4-SNAPSHOT diff --git a/jdbc/pom.xml b/jdbc/pom.xml index 4fc37a7..05fa844 100644 --- a/jdbc/pom.xml +++ b/jdbc/pom.xml @@ -11,7 +11,7 @@ com.kylinolap kylin - 0.6.3 + 0.6.4-SNAPSHOT diff --git a/job/pom.xml b/job/pom.xml index 05cb6ad..a8492ba 100644 --- a/job/pom.xml +++ b/job/pom.xml @@ -4,7 +4,7 @@ com.kylinolap kylin - 0.6.3 + 0.6.4-SNAPSHOT kylin-job diff --git a/metadata/pom.xml b/metadata/pom.xml index a77c88a..8ac56fe 100644 --- a/metadata/pom.xml +++ b/metadata/pom.xml @@ -8,7 +8,7 @@ com.kylinolap kylin - 0.6.3 + 0.6.4-SNAPSHOT diff --git a/pom.xml b/pom.xml index 1e1f606..7751db2 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ com.kylinolap kylin pom - 0.6.3 + 0.6.4-SNAPSHOT Kylin:HadoopOLAPEngine @@ -493,7 +493,7 @@ scm:git:git@github.com:KylinOLAP/Kylin.git scm:git:git@github.com:KylinOLAP/Kylin.git scm:git:git@github.com:KylinOLAP/Kylin.git - kylin-0.6.3 + HEAD diff --git a/query/pom.xml b/query/pom.xml index e0c7f38..ecc5a6f 100644 --- a/query/pom.xml +++ b/query/pom.xml @@ -9,7 +9,7 @@ com.kylinolap kylin - 0.6.3 + 0.6.4-SNAPSHOT diff --git a/server/pom.xml b/server/pom.xml index 9315dfe..9b8342e 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -9,7 +9,7 @@ com.kylinolap kylin - 0.6.3 + 0.6.4-SNAPSHOT diff --git a/storage/pom.xml b/storage/pom.xml index 520fd0e..5d9d4d8 100644 --- a/storage/pom.xml +++ b/storage/pom.xml @@ -7,7 +7,7 @@ com.kylinolap kylin - 0.6.3 + 0.6.4-SNAPSHOT From fb0bc0a8876526a19c5ee3a2656e48a95aa556a0 Mon Sep 17 00:00:00 2001 From: Luke Han Date: Wed, 3 Dec 2014 16:54:31 -0800 Subject: [PATCH 28/35] [maven-release-plugin] rollback the release of kylin-0.6.3 --- atopcalcite/pom.xml | 2 +- common/pom.xml | 2 +- cube/pom.xml | 2 +- dictionary/pom.xml | 2 +- jdbc/pom.xml | 2 +- job/pom.xml | 2 +- metadata/pom.xml | 2 +- pom.xml | 2 +- query/pom.xml | 2 +- server/pom.xml | 2 +- storage/pom.xml | 2 +- 11 files changed, 11 insertions(+), 11 deletions(-) diff --git a/atopcalcite/pom.xml b/atopcalcite/pom.xml index 427927f..e36f434 100644 --- a/atopcalcite/pom.xml +++ b/atopcalcite/pom.xml @@ -8,7 +8,7 @@ com.kylinolap kylin - 0.6.4-SNAPSHOT + 0.6.3-SNAPSHOT diff --git a/common/pom.xml b/common/pom.xml index d7f729d..32a46c6 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -8,7 +8,7 @@ com.kylinolap kylin - 0.6.4-SNAPSHOT + 0.6.3-SNAPSHOT diff --git a/cube/pom.xml b/cube/pom.xml index 6004999..f177c07 100644 --- a/cube/pom.xml +++ b/cube/pom.xml @@ -7,7 +7,7 @@ com.kylinolap kylin - 0.6.4-SNAPSHOT + 0.6.3-SNAPSHOT diff --git a/dictionary/pom.xml b/dictionary/pom.xml index 2e00b70..1d09903 100644 --- a/dictionary/pom.xml +++ b/dictionary/pom.xml @@ -8,7 +8,7 @@ com.kylinolap kylin - 0.6.4-SNAPSHOT + 0.6.3-SNAPSHOT diff --git a/jdbc/pom.xml b/jdbc/pom.xml index 05fa844..e69cac0 100644 --- a/jdbc/pom.xml +++ b/jdbc/pom.xml @@ -11,7 +11,7 @@ com.kylinolap kylin - 0.6.4-SNAPSHOT + 0.6.3-SNAPSHOT diff --git a/job/pom.xml b/job/pom.xml index a8492ba..b039957 100644 --- a/job/pom.xml +++ b/job/pom.xml @@ -4,7 +4,7 @@ com.kylinolap kylin - 0.6.4-SNAPSHOT + 0.6.3-SNAPSHOT kylin-job diff --git a/metadata/pom.xml b/metadata/pom.xml index 8ac56fe..3e76619 100644 --- a/metadata/pom.xml +++ b/metadata/pom.xml @@ -8,7 +8,7 @@ com.kylinolap kylin - 0.6.4-SNAPSHOT + 0.6.3-SNAPSHOT diff --git a/pom.xml b/pom.xml index 7751db2..c38a8cc 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ com.kylinolap kylin pom - 0.6.4-SNAPSHOT + 0.6.3-SNAPSHOT Kylin:HadoopOLAPEngine diff --git a/query/pom.xml b/query/pom.xml index ecc5a6f..0f035b3 100644 --- a/query/pom.xml +++ b/query/pom.xml @@ -9,7 +9,7 @@ com.kylinolap kylin - 0.6.4-SNAPSHOT + 0.6.3-SNAPSHOT diff --git a/server/pom.xml b/server/pom.xml index 9b8342e..fea3397 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -9,7 +9,7 @@ com.kylinolap kylin - 0.6.4-SNAPSHOT + 0.6.3-SNAPSHOT diff --git a/storage/pom.xml b/storage/pom.xml index 5d9d4d8..46c43e5 100644 --- a/storage/pom.xml +++ b/storage/pom.xml @@ -7,7 +7,7 @@ com.kylinolap kylin - 0.6.4-SNAPSHOT + 0.6.3-SNAPSHOT From 059589f7b4ca62a1c5704f6cef6c8ae221097dd9 Mon Sep 17 00:00:00 2001 From: Luke Han Date: Thu, 4 Dec 2014 06:36:30 -0800 Subject: [PATCH 29/35] [maven-release-plugin] prepare release kylin-0.6.3 --- atopcalcite/pom.xml | 2 +- common/pom.xml | 2 +- cube/pom.xml | 2 +- dictionary/pom.xml | 2 +- jdbc/pom.xml | 2 +- job/pom.xml | 2 +- metadata/pom.xml | 2 +- pom.xml | 4 ++-- query/pom.xml | 2 +- server/pom.xml | 2 +- storage/pom.xml | 2 +- 11 files changed, 12 insertions(+), 12 deletions(-) diff --git a/atopcalcite/pom.xml b/atopcalcite/pom.xml index e36f434..aab0f7f 100644 --- a/atopcalcite/pom.xml +++ b/atopcalcite/pom.xml @@ -8,7 +8,7 @@ com.kylinolap kylin - 0.6.3-SNAPSHOT + 0.6.3 diff --git a/common/pom.xml b/common/pom.xml index 32a46c6..53c7f4e 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -8,7 +8,7 @@ com.kylinolap kylin - 0.6.3-SNAPSHOT + 0.6.3 diff --git a/cube/pom.xml b/cube/pom.xml index f177c07..490a817 100644 --- a/cube/pom.xml +++ b/cube/pom.xml @@ -7,7 +7,7 @@ com.kylinolap kylin - 0.6.3-SNAPSHOT + 0.6.3 diff --git a/dictionary/pom.xml b/dictionary/pom.xml index 1d09903..be1ced3 100644 --- a/dictionary/pom.xml +++ b/dictionary/pom.xml @@ -8,7 +8,7 @@ com.kylinolap kylin - 0.6.3-SNAPSHOT + 0.6.3 diff --git a/jdbc/pom.xml b/jdbc/pom.xml index e69cac0..4fc37a7 100644 --- a/jdbc/pom.xml +++ b/jdbc/pom.xml @@ -11,7 +11,7 @@ com.kylinolap kylin - 0.6.3-SNAPSHOT + 0.6.3 diff --git a/job/pom.xml b/job/pom.xml index b039957..05cb6ad 100644 --- a/job/pom.xml +++ b/job/pom.xml @@ -4,7 +4,7 @@ com.kylinolap kylin - 0.6.3-SNAPSHOT + 0.6.3 kylin-job diff --git a/metadata/pom.xml b/metadata/pom.xml index 3e76619..a77c88a 100644 --- a/metadata/pom.xml +++ b/metadata/pom.xml @@ -8,7 +8,7 @@ com.kylinolap kylin - 0.6.3-SNAPSHOT + 0.6.3 diff --git a/pom.xml b/pom.xml index c38a8cc..1e1f606 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ com.kylinolap kylin pom - 0.6.3-SNAPSHOT + 0.6.3 Kylin:HadoopOLAPEngine @@ -493,7 +493,7 @@ scm:git:git@github.com:KylinOLAP/Kylin.git scm:git:git@github.com:KylinOLAP/Kylin.git scm:git:git@github.com:KylinOLAP/Kylin.git - HEAD + kylin-0.6.3 diff --git a/query/pom.xml b/query/pom.xml index 0f035b3..e0c7f38 100644 --- a/query/pom.xml +++ b/query/pom.xml @@ -9,7 +9,7 @@ com.kylinolap kylin - 0.6.3-SNAPSHOT + 0.6.3 diff --git a/server/pom.xml b/server/pom.xml index fea3397..9315dfe 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -9,7 +9,7 @@ com.kylinolap kylin - 0.6.3-SNAPSHOT + 0.6.3 diff --git a/storage/pom.xml b/storage/pom.xml index 46c43e5..520fd0e 100644 --- a/storage/pom.xml +++ b/storage/pom.xml @@ -7,7 +7,7 @@ com.kylinolap kylin - 0.6.3-SNAPSHOT + 0.6.3 From d19da5e96b74db6d3f6adaf4b0fcb95e937a1001 Mon Sep 17 00:00:00 2001 From: Luke Han Date: Thu, 4 Dec 2014 06:40:30 -0800 Subject: [PATCH 30/35] [maven-release-plugin] prepare for next development iteration --- atopcalcite/pom.xml | 2 +- common/pom.xml | 2 +- cube/pom.xml | 2 +- dictionary/pom.xml | 2 +- jdbc/pom.xml | 2 +- job/pom.xml | 2 +- metadata/pom.xml | 2 +- pom.xml | 4 ++-- query/pom.xml | 2 +- server/pom.xml | 2 +- storage/pom.xml | 2 +- 11 files changed, 12 insertions(+), 12 deletions(-) diff --git a/atopcalcite/pom.xml b/atopcalcite/pom.xml index aab0f7f..427927f 100644 --- a/atopcalcite/pom.xml +++ b/atopcalcite/pom.xml @@ -8,7 +8,7 @@ com.kylinolap kylin - 0.6.3 + 0.6.4-SNAPSHOT diff --git a/common/pom.xml b/common/pom.xml index 53c7f4e..d7f729d 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -8,7 +8,7 @@ com.kylinolap kylin - 0.6.3 + 0.6.4-SNAPSHOT diff --git a/cube/pom.xml b/cube/pom.xml index 490a817..6004999 100644 --- a/cube/pom.xml +++ b/cube/pom.xml @@ -7,7 +7,7 @@ com.kylinolap kylin - 0.6.3 + 0.6.4-SNAPSHOT diff --git a/dictionary/pom.xml b/dictionary/pom.xml index be1ced3..2e00b70 100644 --- a/dictionary/pom.xml +++ b/dictionary/pom.xml @@ -8,7 +8,7 @@ com.kylinolap kylin - 0.6.3 + 0.6.4-SNAPSHOT diff --git a/jdbc/pom.xml b/jdbc/pom.xml index 4fc37a7..05fa844 100644 --- a/jdbc/pom.xml +++ b/jdbc/pom.xml @@ -11,7 +11,7 @@ com.kylinolap kylin - 0.6.3 + 0.6.4-SNAPSHOT diff --git a/job/pom.xml b/job/pom.xml index 05cb6ad..a8492ba 100644 --- a/job/pom.xml +++ b/job/pom.xml @@ -4,7 +4,7 @@ com.kylinolap kylin - 0.6.3 + 0.6.4-SNAPSHOT kylin-job diff --git a/metadata/pom.xml b/metadata/pom.xml index a77c88a..8ac56fe 100644 --- a/metadata/pom.xml +++ b/metadata/pom.xml @@ -8,7 +8,7 @@ com.kylinolap kylin - 0.6.3 + 0.6.4-SNAPSHOT diff --git a/pom.xml b/pom.xml index 1e1f606..7751db2 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ com.kylinolap kylin pom - 0.6.3 + 0.6.4-SNAPSHOT Kylin:HadoopOLAPEngine @@ -493,7 +493,7 @@ scm:git:git@github.com:KylinOLAP/Kylin.git scm:git:git@github.com:KylinOLAP/Kylin.git scm:git:git@github.com:KylinOLAP/Kylin.git - kylin-0.6.3 + HEAD diff --git a/query/pom.xml b/query/pom.xml index e0c7f38..ecc5a6f 100644 --- a/query/pom.xml +++ b/query/pom.xml @@ -9,7 +9,7 @@ com.kylinolap kylin - 0.6.3 + 0.6.4-SNAPSHOT diff --git a/server/pom.xml b/server/pom.xml index 9315dfe..9b8342e 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -9,7 +9,7 @@ com.kylinolap kylin - 0.6.3 + 0.6.4-SNAPSHOT diff --git a/storage/pom.xml b/storage/pom.xml index 520fd0e..5d9d4d8 100644 --- a/storage/pom.xml +++ b/storage/pom.xml @@ -7,7 +7,7 @@ com.kylinolap kylin - 0.6.3 + 0.6.4-SNAPSHOT From c4133ebf0709620221a25d778567af2bded0c8c8 Mon Sep 17 00:00:00 2001 From: Luke Han Date: Thu, 4 Dec 2014 06:46:35 -0800 Subject: [PATCH 31/35] [maven-release-plugin] rollback the release of kylin-0.6.3 --- atopcalcite/pom.xml | 2 +- common/pom.xml | 2 +- cube/pom.xml | 2 +- dictionary/pom.xml | 2 +- jdbc/pom.xml | 2 +- job/pom.xml | 2 +- metadata/pom.xml | 2 +- pom.xml | 2 +- query/pom.xml | 2 +- server/pom.xml | 2 +- storage/pom.xml | 2 +- 11 files changed, 11 insertions(+), 11 deletions(-) diff --git a/atopcalcite/pom.xml b/atopcalcite/pom.xml index 427927f..e36f434 100644 --- a/atopcalcite/pom.xml +++ b/atopcalcite/pom.xml @@ -8,7 +8,7 @@ com.kylinolap kylin - 0.6.4-SNAPSHOT + 0.6.3-SNAPSHOT diff --git a/common/pom.xml b/common/pom.xml index d7f729d..32a46c6 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -8,7 +8,7 @@ com.kylinolap kylin - 0.6.4-SNAPSHOT + 0.6.3-SNAPSHOT diff --git a/cube/pom.xml b/cube/pom.xml index 6004999..f177c07 100644 --- a/cube/pom.xml +++ b/cube/pom.xml @@ -7,7 +7,7 @@ com.kylinolap kylin - 0.6.4-SNAPSHOT + 0.6.3-SNAPSHOT diff --git a/dictionary/pom.xml b/dictionary/pom.xml index 2e00b70..1d09903 100644 --- a/dictionary/pom.xml +++ b/dictionary/pom.xml @@ -8,7 +8,7 @@ com.kylinolap kylin - 0.6.4-SNAPSHOT + 0.6.3-SNAPSHOT diff --git a/jdbc/pom.xml b/jdbc/pom.xml index 05fa844..e69cac0 100644 --- a/jdbc/pom.xml +++ b/jdbc/pom.xml @@ -11,7 +11,7 @@ com.kylinolap kylin - 0.6.4-SNAPSHOT + 0.6.3-SNAPSHOT diff --git a/job/pom.xml b/job/pom.xml index a8492ba..b039957 100644 --- a/job/pom.xml +++ b/job/pom.xml @@ -4,7 +4,7 @@ com.kylinolap kylin - 0.6.4-SNAPSHOT + 0.6.3-SNAPSHOT kylin-job diff --git a/metadata/pom.xml b/metadata/pom.xml index 8ac56fe..3e76619 100644 --- a/metadata/pom.xml +++ b/metadata/pom.xml @@ -8,7 +8,7 @@ com.kylinolap kylin - 0.6.4-SNAPSHOT + 0.6.3-SNAPSHOT diff --git a/pom.xml b/pom.xml index 7751db2..c38a8cc 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ com.kylinolap kylin pom - 0.6.4-SNAPSHOT + 0.6.3-SNAPSHOT Kylin:HadoopOLAPEngine diff --git a/query/pom.xml b/query/pom.xml index ecc5a6f..0f035b3 100644 --- a/query/pom.xml +++ b/query/pom.xml @@ -9,7 +9,7 @@ com.kylinolap kylin - 0.6.4-SNAPSHOT + 0.6.3-SNAPSHOT diff --git a/server/pom.xml b/server/pom.xml index 9b8342e..fea3397 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -9,7 +9,7 @@ com.kylinolap kylin - 0.6.4-SNAPSHOT + 0.6.3-SNAPSHOT diff --git a/storage/pom.xml b/storage/pom.xml index 5d9d4d8..46c43e5 100644 --- a/storage/pom.xml +++ b/storage/pom.xml @@ -7,7 +7,7 @@ com.kylinolap kylin - 0.6.4-SNAPSHOT + 0.6.3-SNAPSHOT From 804e8ef6bcdfdbf38e9eee6abf9c5387854d1bbf Mon Sep 17 00:00:00 2001 From: abansal Date: Thu, 4 Dec 2014 11:16:53 -0800 Subject: [PATCH 32/35] Adding Docker installation related instructions --- README.md | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 69d8a61..770a931 100644 --- a/README.md +++ b/README.md @@ -36,9 +36,16 @@ _Tested with Hortonworks HDP 2.1.3 and Cloudera Quickstart VM 5.1._ ### Installation ### -Depending on different use cases Kylin can be installed differently. There are two major scenarios: +It is very easy to install Kylin for exploration/development. There are 3 possible ways: -#### On-Hadoop-CLI installation #### +1. Quick Start (Docker) +2. Sandbox (HDP or CDH sandbox) +3. Dev Environment (IDE + Sandbox) + +#### Quick Start using Docker #### + + +#### Sandbox (On-Hadoop-CLI installation) #### If you are free to install Kylin on your hadoop CLI machine or Hadoop sandbox, this is the most convenient scenario, for it puts everything in a single machine. @@ -50,7 +57,7 @@ For normal users, the application in the above picture means `Kylin Web`, which For a hands-on tutorial please visit [On-Hadoop-CLI installation](https://github.com/KylinOLAP/Kylin/wiki/On-Hadoop-CLI-installation). -#### Off-Hadoop-CLI Installation #### +#### Dev Environment (Off-Hadoop-CLI Installation) #### This is typically for development environment setup. From fdddaa7460e4e9eec57b9afc315c583995d800e5 Mon Sep 17 00:00:00 2001 From: Ankur Bansal Date: Thu, 4 Dec 2014 11:47:29 -0800 Subject: [PATCH 33/35] Docker related documentation --- README.md | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 770a931..3110cea 100644 --- a/README.md +++ b/README.md @@ -34,16 +34,22 @@ _Tested with Hortonworks HDP 2.1.3 and Cloudera Quickstart VM 5.1._ - For the quick tutorial, we assume that your hadoop has priviledges disabled which means any user could run hadoop and hive command -### Installation ### - -It is very easy to install Kylin for exploration/development. There are 3 possible ways: +Installation +------------ +It is very easy to install Kylin for exploration/development. There are 3 supported options: 1. Quick Start (Docker) 2. Sandbox (HDP or CDH sandbox) 3. Dev Environment (IDE + Sandbox) -#### Quick Start using Docker #### +#### Quick Start (using Docker) #### +This is the easiest option to try Kylin. We have created docker container for Kylin (along with Hadoop, HBase and Hive) with help of SequenceIQ. The only thing you will need to do is to pull the container from the official Docker repository to be up and running in few minutes. + +Features: +- Size - Light weight compared to downloading and setting up HDP or CDH sandbox. +- Cluster support - Supports multi node installation. +- Fully Automated - No manual steps. One command does it all #### Sandbox (On-Hadoop-CLI installation) #### From 604a5b70a8467ad63ee4bdcd9dcec86badf17dab Mon Sep 17 00:00:00 2001 From: Ankur Bansal Date: Thu, 4 Dec 2014 11:54:48 -0800 Subject: [PATCH 34/35] cleanup of doc --- README.md | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 3110cea..d792030 100644 --- a/README.md +++ b/README.md @@ -38,8 +38,8 @@ Installation ------------ It is very easy to install Kylin for exploration/development. There are 3 supported options: -1. Quick Start (Docker) -2. Sandbox (HDP or CDH sandbox) +1. Quick Start (Docker) +2. Sandbox (HDP or CDH sandbox) 3. Dev Environment (IDE + Sandbox) #### Quick Start (using Docker) #### @@ -51,6 +51,9 @@ Features: - Cluster support - Supports multi node installation. - Fully Automated - No manual steps. One command does it all +For a hands-on tutorial please visit [Kylin Docker installation](https://github.com/KylinOLAP/Kylin/wiki/On-Hadoop-Kylin-installation-using-Docker). + + #### Sandbox (On-Hadoop-CLI installation) #### If you are free to install Kylin on your hadoop CLI machine or Hadoop sandbox, this is the most convenient scenario, for it puts everything in a single machine. @@ -86,3 +89,5 @@ For a hands-on tutorial please visit [Off-Hadoop-CLI installation](https://githu * Presentation: [Kylin Hadoop OLAP Engine v1.0](https://github.com/KylinOLAP/Kylin/blob/master/docs/Kylin_Hadoop_OLAP_Engine_v1.0.pdf?raw=true) +* Apache Proposal: [Apache Kylin](https://wiki.apache.org/incubator/KylinProposal) + From 7aa4cb5d1fbb58dd93cc6785e8cb266257c96339 Mon Sep 17 00:00:00 2001 From: "qianhao.zhou" Date: Mon, 8 Dec 2014 15:34:54 +0800 Subject: [PATCH 35/35] refactor --- .../test_case_data/localmeta/project/default.json | 8 +- .../localmeta/project/onlyinner.json | 4 +- .../test_case_data/localmeta/project/onlyleft.json | 4 +- .../model/realization/DataModelRealization.java | 10 + .../metadata/project/ProjectDataModel.java | 7 +- .../metadata/project/ProjectInstance.java | 270 +++++++++++++++++++++ .../kylinolap/metadata/project/ProjectManager.java | 264 ++++++++++++++++++++ .../metadata/project/ProjectStatusEnum.java | 22 ++ 8 files changed, 578 insertions(+), 11 deletions(-) create mode 100644 metadata/src/main/java/com/kylinolap/metadata/model/realization/DataModelRealization.java create mode 100644 metadata/src/main/java/com/kylinolap/metadata/project/ProjectInstance.java create mode 100644 metadata/src/main/java/com/kylinolap/metadata/project/ProjectManager.java create mode 100644 metadata/src/main/java/com/kylinolap/metadata/project/ProjectStatusEnum.java diff --git a/examples/test_case_data/localmeta/project/default.json b/examples/test_case_data/localmeta/project/default.json index 0a878d8..730aa9e 100644 --- a/examples/test_case_data/localmeta/project/default.json +++ b/examples/test_case_data/localmeta/project/default.json @@ -4,22 +4,22 @@ "datamodels": [ { "name": "test_kylin_with_slr_model_desc", - "type": "cube", + "type": "CUBE", "realization": "test_kylin_cube_with_slr_empty" }, { "name": "test_kylin_without_slr_model_desc", - "type": "cube", + "type": "CUBE", "realization": "test_kylin_cube_without_slr_empty" }, { "name": "test_kylin_with_slr_left_join_model_desc", - "type": "cube", + "type": "CUBE", "realization": "test_kylin_cube_with_slr_left_join_empty" }, { "name": "test_kylin_without_slr_left_join_model_desc", - "type": "cube", + "type": "CUBE", "realization": "test_kylin_cube_without_slr_left_join_empty" } ], diff --git a/examples/test_case_data/localmeta/project/onlyinner.json b/examples/test_case_data/localmeta/project/onlyinner.json index 1037b9e..daf138a 100644 --- a/examples/test_case_data/localmeta/project/onlyinner.json +++ b/examples/test_case_data/localmeta/project/onlyinner.json @@ -4,12 +4,12 @@ "datamodels": [ { "name": "test_kylin_with_slr_model_desc", - "type": "cube", + "type": "CUBE", "realization": "test_kylin_cube_with_slr_empty" }, { "name": "test_kylin_without_slr_model_desc", - "type": "cube", + "type": "CUBE", "realization": "test_kylin_cube_without_slr_empty" } ], diff --git a/examples/test_case_data/localmeta/project/onlyleft.json b/examples/test_case_data/localmeta/project/onlyleft.json index bb3ef50..c37b31b 100644 --- a/examples/test_case_data/localmeta/project/onlyleft.json +++ b/examples/test_case_data/localmeta/project/onlyleft.json @@ -4,12 +4,12 @@ "datamodels": [ { "name": "test_kylin_with_slr_left_join_model_desc", - "type": "cube", + "type": "CUBE", "realization": "test_kylin_cube_with_slr_left_join_empty" }, { "name": "test_kylin_without_slr_left_join_model_desc", - "type": "cube", + "type": "CUBE", "realization": "test_kylin_cube_without_slr_left_join_empty" } ], diff --git a/metadata/src/main/java/com/kylinolap/metadata/model/realization/DataModelRealization.java b/metadata/src/main/java/com/kylinolap/metadata/model/realization/DataModelRealization.java new file mode 100644 index 0000000..9b4d62b --- /dev/null +++ b/metadata/src/main/java/com/kylinolap/metadata/model/realization/DataModelRealization.java @@ -0,0 +1,10 @@ +package com.kylinolap.metadata.model.realization; + +/** + * Created by qianzhou on 12/8/14. + */ +public enum DataModelRealization { + + CUBE, + INVERTED_INDEX; +} diff --git a/metadata/src/main/java/com/kylinolap/metadata/project/ProjectDataModel.java b/metadata/src/main/java/com/kylinolap/metadata/project/ProjectDataModel.java index e0fd40f..dce36d5 100644 --- a/metadata/src/main/java/com/kylinolap/metadata/project/ProjectDataModel.java +++ b/metadata/src/main/java/com/kylinolap/metadata/project/ProjectDataModel.java @@ -2,6 +2,7 @@ import com.fasterxml.jackson.annotation.JsonAutoDetect; import com.fasterxml.jackson.annotation.JsonProperty; +import com.kylinolap.metadata.model.realization.DataModelRealization; /** * Created by qianzhou on 12/5/14. @@ -13,7 +14,7 @@ private String name; @JsonProperty("type") - private String type; + private DataModelRealization type; @JsonProperty("realization") private String realization; @@ -26,11 +27,11 @@ public void setName(String name) { this.name = name; } - public String getType() { + public DataModelRealization getType() { return type; } - public void setType(String type) { + public void setType(DataModelRealization type) { this.type = type; } diff --git a/metadata/src/main/java/com/kylinolap/metadata/project/ProjectInstance.java b/metadata/src/main/java/com/kylinolap/metadata/project/ProjectInstance.java new file mode 100644 index 0000000..a10a38a --- /dev/null +++ b/metadata/src/main/java/com/kylinolap/metadata/project/ProjectInstance.java @@ -0,0 +1,270 @@ +/* + * Copyright 2013-2014 eBay Software Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.kylinolap.metadata.project; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Predicate; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import com.kylinolap.common.persistence.ResourceStore; +import com.kylinolap.common.persistence.RootPersistentEntity; +import com.kylinolap.metadata.model.realization.DataModelRealization; + +import javax.annotation.Nullable; +import java.util.ArrayList; +import java.util.List; +import java.util.Set; +import java.util.TreeSet; + +/** + * Project is a concept in Kylin similar to schema in DBMS + */ +@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE) +public class ProjectInstance extends RootPersistentEntity { + + public static final String DEFAULT_PROJECT_NAME = "DEFAULT"; + + @JsonProperty("name") + private String name; + + @JsonProperty("cubes") + private List cubes; + + @JsonProperty("tables") + private Set tables; + + + @JsonProperty("owner") + private String owner; + + @JsonProperty("status") + private ProjectStatusEnum status; + + @JsonProperty("create_time") + private String createTime; + + @JsonProperty("last_update_time") + private String lastUpdateTime; + + @JsonProperty("description") + private String description; + + @JsonProperty("datamodels") + private List dataModels; + + public String getResourcePath() { + return concatResourcePath(name); + } + + public static String concatResourcePath(String projectName) { + return ResourceStore.PROJECT_RESOURCE_ROOT + "/" + projectName + ".json"; + } + + public static String getNormalizedProjectName(String project) { + if (project == null) + throw new IllegalStateException("Trying to normalized a project name which is null"); + + return project.toUpperCase(); + } + + // ============================================================================ + + public static ProjectInstance create(String name, String owner, String description, List cubes) { + ProjectInstance projectInstance = new ProjectInstance(); + + projectInstance.updateRandomUuid(); + projectInstance.setName(name); + projectInstance.setOwner(owner); + projectInstance.setDescription(description); + projectInstance.setStatus(ProjectStatusEnum.ENABLED); + projectInstance.setCreateTime(formatTime(System.currentTimeMillis())); + if (cubes != null) + projectInstance.setCubes(cubes); + else + projectInstance.setCubes(new ArrayList()); + + return projectInstance; + } + + public ProjectInstance() { + + } + + public ProjectInstance(String name, List cubes, String owner) { + this.name = name; + this.cubes = cubes; + this.owner = owner; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public ProjectStatusEnum getStatus() { + return status; + } + + public void setStatus(ProjectStatusEnum status) { + this.status = status; + } + + public String getCreateTime() { + return createTime; + } + + public void setCreateTime(String createTime) { + this.createTime = createTime; + } + + public String getName() { + return this.name; + } + + public void setName(String name) { + this.name = name; + } + + public boolean containsCube(String cubeName) { + cubeName = cubeName.toUpperCase(); + return cubes.contains(cubeName); + } + + public void removeCube(String cubeName) { + cubeName = cubeName.toUpperCase(); + cubes.remove(cubeName); + } + + public int getCubesCount() { + return cubes.size(); + } + + public void addCube(String cubeName) { + cubeName = cubeName.toUpperCase(); + this.cubes.add(cubeName); + } + + public List getCubes() { + return cubes; + } + + + public void setCubes(List cubes) { + this.cubes = cubes; + } + + public void setTables(Set tables) { + this.tables = tables; + } + + public boolean containsTable(String tableName) { + tableName = tableName.toUpperCase(); + return tables.contains(tableName); + } + + public void removeTable(String tableName) { + tableName = tableName.toUpperCase(); + tables.remove(tableName); + } + + public int getTablesCount() { + return this.getTables().size(); + } + + public void addTable(String tableName) { + tableName = tableName.toUpperCase(); + this.getTables().add(tableName); + } + + //will return new Set for null + public Set getTables() { + tables = tables == null ? new TreeSet() : tables; + return tables; + } + + public String getOwner() { + return owner; + } + + public void setOwner(String owner) { + this.owner = owner; + } + + public String getLastUpdateTime() { + return lastUpdateTime; + } + + public void setLastUpdateTime(String lastUpdateTime) { + this.lastUpdateTime = lastUpdateTime; + } + + public void recordUpdateTime(long timeMillis) { + this.lastUpdateTime = formatTime(timeMillis); + } + + public List getDataModels() { + return dataModels; + } + + public void setDataModels(List dataModels) { + this.dataModels = dataModels; + } + + public void init() { + if (name == null) + name = ProjectInstance.DEFAULT_PROJECT_NAME; + + if (cubes == null) { + cubes = new ArrayList(); + } + + for (int i = 0; i < cubes.size(); ++i) { + if (cubes.get(i) != null) + cubes.set(i, cubes.get(i).toUpperCase()); + } + } + + public boolean containsRealization(DataModelRealization realization, String name) { + if (dataModels == null) { + return false; + } + for (ProjectDataModel dataModel: dataModels) { + if (dataModel.getType() == realization && dataModel.getName().equalsIgnoreCase(name)) { + return true; + } + } + return false; + } + + public List getDataModels(final DataModelRealization dataModelRealization) { + return ImmutableList.copyOf(Iterables.filter(dataModels, new Predicate() { + @Override + public boolean apply(@Nullable ProjectDataModel input) { + return input.getType() == dataModelRealization; + } + })); + } + + @Override + public String toString() { + return "ProjectDesc [name=" + name + "]"; + } +} diff --git a/metadata/src/main/java/com/kylinolap/metadata/project/ProjectManager.java b/metadata/src/main/java/com/kylinolap/metadata/project/ProjectManager.java new file mode 100644 index 0000000..cc16916 --- /dev/null +++ b/metadata/src/main/java/com/kylinolap/metadata/project/ProjectManager.java @@ -0,0 +1,264 @@ +/* + * Copyright 2013-2014 eBay Software Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kylinolap.metadata.project; + +import com.kylinolap.common.KylinConfig; +import com.kylinolap.common.persistence.JsonSerializer; +import com.kylinolap.common.persistence.ResourceStore; +import com.kylinolap.common.persistence.Serializer; +import com.kylinolap.common.restclient.Broadcaster; +import com.kylinolap.common.restclient.SingleValueCache; +import com.kylinolap.metadata.model.realization.DataModelRealization; +import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ConcurrentHashMap; + +/** + * @author xduo + */ +public class ProjectManager { + private static final Logger logger = LoggerFactory.getLogger(ProjectManager.class); + + // static cached instances + private static final ConcurrentHashMap CACHE = new ConcurrentHashMap(); + private static final Serializer PROJECT_SERIALIZER = new JsonSerializer(ProjectInstance.class); + + private KylinConfig config; + // project name => ProjrectDesc + private SingleValueCache projectMap = new SingleValueCache(Broadcaster.TYPE.PROJECT); + // project name => tables + + public static ProjectManager getInstance(KylinConfig config) { + ProjectManager r = CACHE.get(config); + if (r != null) { + return r; + } + + synchronized (ProjectManager.class) { + r = CACHE.get(config); + if (r != null) { + return r; + } + try { + r = new ProjectManager(config); + CACHE.put(config, r); + if (CACHE.size() > 1) { + logger.warn("More than one singleton exist"); + } + return r; + } catch (IOException e) { + throw new IllegalStateException("Failed to init CubeManager from " + config, e); + } + } + } + + public static synchronized void removeInstance(KylinConfig config) { + CACHE.remove(config); + } + + private ProjectManager(KylinConfig config) throws IOException { + logger.info("Initializing CubeManager with metadata url " + config); + this.config = config; + + loadAllProjects(); + } + + public List listAllProjects() { + return new ArrayList(projectMap.values()); + } + + public List getProjects(String cubeName) { + return this.findProjects(cubeName); + } + + public ProjectInstance dropProject(String projectName) throws IOException { + if (projectName == null) + throw new IllegalArgumentException("Project name not given"); + + ProjectInstance projectInstance = getProject(projectName); + + if (projectInstance == null) { + throw new IllegalStateException("The project named " + projectName + " does not exist"); + } + + if (projectInstance.getCubes().size() != 0) { + throw new IllegalStateException("The project named " + projectName + " can not be deleted because there's still cubes in it. Delete all the cubes first."); + } + + logger.info("Dropping project '" + projectInstance.getName() + "'"); + deleteResource(projectInstance); + + return projectInstance; + } + + public ProjectInstance getProject(String projectName) { + if (projectName == null) + return null; + projectName = ProjectInstance.getNormalizedProjectName(projectName); + return projectMap.get(projectName); + } + + public ProjectInstance createProject(String projectName, String owner, String description) throws IOException { + + logger.info("Creating project '" + projectName); + + ProjectInstance currentProject = getProject(projectName); + if (currentProject == null) { + currentProject = ProjectInstance.create(projectName, owner, description, null); + } else { + throw new IllegalStateException("The project named " + projectName + "already exists"); + } + + saveResource(currentProject); + + return currentProject; + } + + public ProjectInstance updateProject(ProjectInstance project, String newName, String newDesc) throws IOException { + if (!project.getName().equals(newName)) { + ProjectInstance newProject = this.createProject(newName, project.getOwner(), newDesc); + newProject.setCreateTime(project.getCreateTime()); + newProject.recordUpdateTime(System.currentTimeMillis()); + newProject.setCubes(project.getCubes()); + + deleteResource(project); + saveResource(newProject); + + return newProject; + } else { + project.setName(newName); + project.setDescription(newDesc); + + if (project.getUuid() == null) { + project.updateRandomUuid(); + } + + saveResource(project); + + return project; + } + } + + public boolean containsRealization(String projectName, DataModelRealization realization, String name) { + ProjectInstance project = getProject(projectName); + if (project == null) { + return false; + } + return project.containsRealization(realization, name); + } + + public void loadProjectCache(ProjectInstance project, boolean triggerUpdate) throws IOException { + loadProject(project.getResourcePath(), triggerUpdate); + } + + public void removeProjectCache(ProjectInstance project) { + String projectName = ProjectInstance.getNormalizedProjectName(project.getName()); + if (projectMap.containsKey(projectName)) { + projectMap.remove(projectName); + } + } + + private List findProjects(String cubeName) { + List projects = new ArrayList(); + for (ProjectInstance projectInstance : projectMap.values()) { + if (projectInstance.containsCube(cubeName)) { + projects.add(projectInstance); + } + } + + return projects; + } + + private synchronized ProjectInstance loadProject(String path, boolean triggerUpdate) throws IOException { + ResourceStore store = getStore(); + logger.debug("Loading CubeInstance " + store.getReadableResourcePath(path)); + + ProjectInstance projectInstance = store.getResource(path, ProjectInstance.class, PROJECT_SERIALIZER); + projectInstance.init(); + + if (StringUtils.isBlank(projectInstance.getName())) { + throw new IllegalStateException("Project name must not be blank"); + } + + if (triggerUpdate) { + projectMap.put(projectInstance.getName().toUpperCase(), projectInstance); + } else { + projectMap.putLocal(projectInstance.getName().toUpperCase(), projectInstance); + } + + return projectInstance; + } + + private void loadAllProjects() throws IOException { + ResourceStore store = getStore(); + List paths = store.collectResourceRecursively(ResourceStore.PROJECT_RESOURCE_ROOT, ".json"); + + logger.debug("Loading Project from folder " + store.getReadableResourcePath(ResourceStore.PROJECT_RESOURCE_ROOT)); + + for (String path : paths) { + loadProject(path, false); + } + + logger.debug("Loaded " + paths.size() + " Project(s)"); + } + + private ProjectInstance addCubeToProject(String cubeName, String project, String user) throws IOException { + String newProjectName = ProjectInstance.getNormalizedProjectName(project); + ProjectInstance newProject = getProject(newProjectName); + if (newProject == null) { + newProject = this.createProject(newProjectName, user, "This is a project automatically added when adding cube " + cubeName); + } + newProject.addCube(cubeName); + saveResource(newProject); + + return newProject; + } + + private void saveResource(ProjectInstance proj) throws IOException { + ResourceStore store = getStore(); + store.putResource(proj.getResourcePath(), proj, PROJECT_SERIALIZER); + afterProjectUpdated(proj); + } + + private void deleteResource(ProjectInstance proj) throws IOException { + ResourceStore store = getStore(); + store.deleteResource(proj.getResourcePath()); + this.afterProjectDropped(proj); + } + + private void afterProjectUpdated(ProjectInstance updatedProject) { + try { + this.loadProjectCache(updatedProject, true); + } catch (IOException e) { + logger.error(e.getLocalizedMessage(), e); + } + } + + private void afterProjectDropped(ProjectInstance droppedProject) { + this.removeProjectCache(droppedProject); + } + + private ResourceStore getStore() { + return ResourceStore.getStore(this.config); + } + +} diff --git a/metadata/src/main/java/com/kylinolap/metadata/project/ProjectStatusEnum.java b/metadata/src/main/java/com/kylinolap/metadata/project/ProjectStatusEnum.java new file mode 100644 index 0000000..541cbae --- /dev/null +++ b/metadata/src/main/java/com/kylinolap/metadata/project/ProjectStatusEnum.java @@ -0,0 +1,22 @@ +/* + * Copyright 2013-2014 eBay Software Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.kylinolap.metadata.project; + +public enum ProjectStatusEnum { + + DISABLED, ENABLED + +}