From 5feae69b733ab21f0d87d03e358853f02c900928 Mon Sep 17 00:00:00 2001 From: "qianhao.zhou" Date: Mon, 8 Dec 2014 16:29:46 +0800 Subject: [PATCH 1/4] refactor --- .../kylinolap/cube/project/ProjectInstance.java | 245 --------------------- .../com/kylinolap/cube/project/ProjectManager.java | 1 + .../kylinolap/cube/project/ProjectStatusEnum.java | 22 -- .../model/realization/DataModelRealization.java | 10 - .../realization/DataModelRealizationType.java | 10 + .../metadata/project/ProjectDataModel.java | 8 +- .../metadata/project/ProjectInstance.java | 8 +- .../kylinolap/metadata/project/ProjectManager.java | 15 +- .../com/kylinolap/rest/service/ProjectService.java | 2 +- 9 files changed, 33 insertions(+), 288 deletions(-) delete mode 100644 cube/src/main/java/com/kylinolap/cube/project/ProjectInstance.java delete mode 100644 cube/src/main/java/com/kylinolap/cube/project/ProjectStatusEnum.java delete mode 100644 metadata/src/main/java/com/kylinolap/metadata/model/realization/DataModelRealization.java create mode 100644 metadata/src/main/java/com/kylinolap/metadata/model/realization/DataModelRealizationType.java diff --git a/cube/src/main/java/com/kylinolap/cube/project/ProjectInstance.java b/cube/src/main/java/com/kylinolap/cube/project/ProjectInstance.java deleted file mode 100644 index 17b0ed1..0000000 --- a/cube/src/main/java/com/kylinolap/cube/project/ProjectInstance.java +++ /dev/null @@ -1,245 +0,0 @@ -/* - * Copyright 2013-2014 eBay Software Foundation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.kylinolap.cube.project; - -import java.util.ArrayList; -import java.util.List; -import java.util.Set; -import java.util.TreeSet; - -import com.fasterxml.jackson.annotation.JsonAutoDetect; -import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.kylinolap.common.persistence.ResourceStore; -import com.kylinolap.common.persistence.RootPersistentEntity; -import com.kylinolap.metadata.project.ProjectDataModel; - -/** - * Project is a concept in Kylin similar to schema in DBMS - */ -@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE) -public class ProjectInstance extends RootPersistentEntity { - - public static final String DEFAULT_PROJECT_NAME = "DEFAULT"; - - @JsonProperty("name") - private String name; - - @JsonProperty("cubes") - private List cubes; - - @JsonProperty("tables") - private Set tables; - - - @JsonProperty("owner") - private String owner; - - @JsonProperty("status") - private ProjectStatusEnum status; - - @JsonProperty("create_time") - private String createTime; - - @JsonProperty("last_update_time") - private String lastUpdateTime; - - @JsonProperty("description") - private String description; - - @JsonProperty("datamodels") - private List dataModels; - - public String getResourcePath() { - return concatResourcePath(name); - } - - public static String concatResourcePath(String projectName) { - return ResourceStore.PROJECT_RESOURCE_ROOT + "/" + projectName + ".json"; - } - - public static String getNormalizedProjectName(String project) { - if (project == null) - throw new IllegalStateException("Trying to normalized a project name which is null"); - - return project.toUpperCase(); - } - - // ============================================================================ - - public static ProjectInstance create(String name, String owner, String description, List cubes) { - ProjectInstance projectInstance = new ProjectInstance(); - - projectInstance.updateRandomUuid(); - projectInstance.setName(name); - projectInstance.setOwner(owner); - projectInstance.setDescription(description); - projectInstance.setStatus(ProjectStatusEnum.ENABLED); - projectInstance.setCreateTime(formatTime(System.currentTimeMillis())); - if (cubes != null) - projectInstance.setCubes(cubes); - else - projectInstance.setCubes(new ArrayList()); - - return projectInstance; - } - - public ProjectInstance() { - - } - - public ProjectInstance(String name, List cubes, String owner) { - this.name = name; - this.cubes = cubes; - this.owner = owner; - } - - public String getDescription() { - return description; - } - - public void setDescription(String description) { - this.description = description; - } - - public ProjectStatusEnum getStatus() { - return status; - } - - public void setStatus(ProjectStatusEnum status) { - this.status = status; - } - - public String getCreateTime() { - return createTime; - } - - public void setCreateTime(String createTime) { - this.createTime = createTime; - } - - public String getName() { - return this.name; - } - - public void setName(String name) { - this.name = name; - } - - public boolean containsCube(String cubeName) { - cubeName = cubeName.toUpperCase(); - return cubes.contains(cubeName); - } - - public void removeCube(String cubeName) { - cubeName = cubeName.toUpperCase(); - cubes.remove(cubeName); - } - - public int getCubesCount() { - return cubes.size(); - } - - public void addCube(String cubeName) { - cubeName = cubeName.toUpperCase(); - this.cubes.add(cubeName); - } - - public List getCubes() { - return cubes; - } - - - public void setCubes(List cubes) { - this.cubes = cubes; - } - - public void setTables(Set tables) { - this.tables = tables; - } - - public boolean containsTable(String tableName) { - tableName = tableName.toUpperCase(); - return tables.contains(tableName); - } - - public void removeTable(String tableName) { - tableName = tableName.toUpperCase(); - tables.remove(tableName); - } - - public int getTablesCount() { - return this.getTables().size(); - } - - public void addTable(String tableName) { - tableName = tableName.toUpperCase(); - this.getTables().add(tableName); - } - - //will return new Set for null - public Set getTables() { - tables = tables == null ? new TreeSet() : tables; - return tables; - } - - public String getOwner() { - return owner; - } - - public void setOwner(String owner) { - this.owner = owner; - } - - public String getLastUpdateTime() { - return lastUpdateTime; - } - - public void setLastUpdateTime(String lastUpdateTime) { - this.lastUpdateTime = lastUpdateTime; - } - - public void recordUpdateTime(long timeMillis) { - this.lastUpdateTime = formatTime(timeMillis); - } - - public List getDataModels() { - return dataModels; - } - - public void setDataModels(List dataModels) { - this.dataModels = dataModels; - } - - public void init() { - if (name == null) - name = ProjectInstance.DEFAULT_PROJECT_NAME; - - if (cubes == null) { - cubes = new ArrayList(); - } - - for (int i = 0; i < cubes.size(); ++i) { - if (cubes.get(i) != null) - cubes.set(i, cubes.get(i).toUpperCase()); - } - } - - @Override - public String toString() { - return "ProjectDesc [name=" + name + "]"; - } -} diff --git a/cube/src/main/java/com/kylinolap/cube/project/ProjectManager.java b/cube/src/main/java/com/kylinolap/cube/project/ProjectManager.java index 979f773..ccdd31d 100644 --- a/cube/src/main/java/com/kylinolap/cube/project/ProjectManager.java +++ b/cube/src/main/java/com/kylinolap/cube/project/ProjectManager.java @@ -20,6 +20,7 @@ import java.util.*; import java.util.concurrent.ConcurrentHashMap; +import com.kylinolap.metadata.project.ProjectInstance; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/cube/src/main/java/com/kylinolap/cube/project/ProjectStatusEnum.java b/cube/src/main/java/com/kylinolap/cube/project/ProjectStatusEnum.java deleted file mode 100644 index 0c582a7..0000000 --- a/cube/src/main/java/com/kylinolap/cube/project/ProjectStatusEnum.java +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Copyright 2013-2014 eBay Software Foundation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.kylinolap.cube.project; - -public enum ProjectStatusEnum { - - DISABLED, ENABLED - -} diff --git a/metadata/src/main/java/com/kylinolap/metadata/model/realization/DataModelRealization.java b/metadata/src/main/java/com/kylinolap/metadata/model/realization/DataModelRealization.java deleted file mode 100644 index 9b4d62b..0000000 --- a/metadata/src/main/java/com/kylinolap/metadata/model/realization/DataModelRealization.java +++ /dev/null @@ -1,10 +0,0 @@ -package com.kylinolap.metadata.model.realization; - -/** - * Created by qianzhou on 12/8/14. - */ -public enum DataModelRealization { - - CUBE, - INVERTED_INDEX; -} diff --git a/metadata/src/main/java/com/kylinolap/metadata/model/realization/DataModelRealizationType.java b/metadata/src/main/java/com/kylinolap/metadata/model/realization/DataModelRealizationType.java new file mode 100644 index 0000000..defa1ce --- /dev/null +++ b/metadata/src/main/java/com/kylinolap/metadata/model/realization/DataModelRealizationType.java @@ -0,0 +1,10 @@ +package com.kylinolap.metadata.model.realization; + +/** + * Created by qianzhou on 12/8/14. + */ +public enum DataModelRealizationType { + + CUBE, + INVERTED_INDEX +} diff --git a/metadata/src/main/java/com/kylinolap/metadata/project/ProjectDataModel.java b/metadata/src/main/java/com/kylinolap/metadata/project/ProjectDataModel.java index dce36d5..d6b0664 100644 --- a/metadata/src/main/java/com/kylinolap/metadata/project/ProjectDataModel.java +++ b/metadata/src/main/java/com/kylinolap/metadata/project/ProjectDataModel.java @@ -2,7 +2,7 @@ import com.fasterxml.jackson.annotation.JsonAutoDetect; import com.fasterxml.jackson.annotation.JsonProperty; -import com.kylinolap.metadata.model.realization.DataModelRealization; +import com.kylinolap.metadata.model.realization.DataModelRealizationType; /** * Created by qianzhou on 12/5/14. @@ -14,7 +14,7 @@ private String name; @JsonProperty("type") - private DataModelRealization type; + private DataModelRealizationType type; @JsonProperty("realization") private String realization; @@ -27,11 +27,11 @@ public void setName(String name) { this.name = name; } - public DataModelRealization getType() { + public DataModelRealizationType getType() { return type; } - public void setType(DataModelRealization type) { + public void setType(DataModelRealizationType type) { this.type = type; } diff --git a/metadata/src/main/java/com/kylinolap/metadata/project/ProjectInstance.java b/metadata/src/main/java/com/kylinolap/metadata/project/ProjectInstance.java index a10a38a..00c5041 100644 --- a/metadata/src/main/java/com/kylinolap/metadata/project/ProjectInstance.java +++ b/metadata/src/main/java/com/kylinolap/metadata/project/ProjectInstance.java @@ -23,7 +23,7 @@ import com.google.common.collect.Iterables; import com.kylinolap.common.persistence.ResourceStore; import com.kylinolap.common.persistence.RootPersistentEntity; -import com.kylinolap.metadata.model.realization.DataModelRealization; +import com.kylinolap.metadata.model.realization.DataModelRealizationType; import javax.annotation.Nullable; import java.util.ArrayList; @@ -242,7 +242,7 @@ public void init() { } } - public boolean containsRealization(DataModelRealization realization, String name) { + public boolean containsRealization(DataModelRealizationType realization, String name) { if (dataModels == null) { return false; } @@ -254,11 +254,11 @@ public boolean containsRealization(DataModelRealization realization, String name return false; } - public List getDataModels(final DataModelRealization dataModelRealization) { + public List getDataModels(final DataModelRealizationType dataModelRealizationType) { return ImmutableList.copyOf(Iterables.filter(dataModels, new Predicate() { @Override public boolean apply(@Nullable ProjectDataModel input) { - return input.getType() == dataModelRealization; + return input.getType() == dataModelRealizationType; } })); } diff --git a/metadata/src/main/java/com/kylinolap/metadata/project/ProjectManager.java b/metadata/src/main/java/com/kylinolap/metadata/project/ProjectManager.java index cc16916..db4b394 100644 --- a/metadata/src/main/java/com/kylinolap/metadata/project/ProjectManager.java +++ b/metadata/src/main/java/com/kylinolap/metadata/project/ProjectManager.java @@ -16,13 +16,15 @@ package com.kylinolap.metadata.project; +import com.google.common.collect.Maps; import com.kylinolap.common.KylinConfig; import com.kylinolap.common.persistence.JsonSerializer; import com.kylinolap.common.persistence.ResourceStore; import com.kylinolap.common.persistence.Serializer; import com.kylinolap.common.restclient.Broadcaster; import com.kylinolap.common.restclient.SingleValueCache; -import com.kylinolap.metadata.model.realization.DataModelRealization; +import com.kylinolap.metadata.model.realization.DataModelRealizationType; +import com.kylinolap.metadata.model.realization.IDataModelRealization; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -31,6 +33,7 @@ import java.util.ArrayList; import java.util.List; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; /** * @author xduo @@ -41,6 +44,7 @@ // static cached instances private static final ConcurrentHashMap CACHE = new ConcurrentHashMap(); private static final Serializer PROJECT_SERIALIZER = new JsonSerializer(ProjectInstance.class); + private ConcurrentMap, Class> realizations = Maps.newConcurrentMap(); private KylinConfig config; // project name => ProjrectDesc @@ -82,6 +86,13 @@ private ProjectManager(KylinConfig config) throws IOException { loadAllProjects(); } + public void registerDataModelRealization(Class realization) { + if (realization == null) { + throw new NullPointerException("realization cannot be null"); + } + realizations.putIfAbsent(realization, realization); + } + public List listAllProjects() { return new ArrayList(projectMap.values()); } @@ -158,7 +169,7 @@ public ProjectInstance updateProject(ProjectInstance project, String newName, St } } - public boolean containsRealization(String projectName, DataModelRealization realization, String name) { + public boolean containsRealization(String projectName, DataModelRealizationType realization, String name) { ProjectInstance project = getProject(projectName); if (project == null) { return false; diff --git a/server/src/main/java/com/kylinolap/rest/service/ProjectService.java b/server/src/main/java/com/kylinolap/rest/service/ProjectService.java index 73c7302..a170167 100644 --- a/server/src/main/java/com/kylinolap/rest/service/ProjectService.java +++ b/server/src/main/java/com/kylinolap/rest/service/ProjectService.java @@ -20,6 +20,7 @@ import java.util.Collections; import java.util.List; +import com.kylinolap.metadata.project.ProjectInstance; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -27,7 +28,6 @@ import org.springframework.security.core.context.SecurityContextHolder; import org.springframework.stereotype.Component; -import com.kylinolap.cube.project.ProjectInstance; import com.kylinolap.rest.constant.Constant; import com.kylinolap.rest.exception.InternalErrorException; import com.kylinolap.rest.request.CreateProjectRequest; From 37928259c03920b5fb1abab3ddfa85cdcd439620 Mon Sep 17 00:00:00 2001 From: "qianhao.zhou" Date: Mon, 8 Dec 2014 16:43:51 +0800 Subject: [PATCH 2/4] fix compile issue --- .../main/java/com/kylinolap/cube/CubeManager.java | 2 +- .../java/com/kylinolap/cube/CubeManagerTest.java | 2 +- .../com/kylinolap/cube/SegmentManagementTest.java | 2 +- .../kylinolap/cube/project/ProjectManagerTest.java | 1 + .../main/java/com/kylinolap/job/JobManager.java | 30 +++++++++------------- .../com/kylinolap/job/tools/CubeMigrationCLI.java | 3 +-- .../metadata/project/ProjectInstance.java | 1 + .../com/kylinolap/query/schema/OLAPSchema.java | 2 +- .../kylinolap/query/schema/OLAPSchemaFactory.java | 2 +- .../com/kylinolap/query/test/KylinQueryTest.java | 2 +- .../kylinolap/rest/controller/CubeController.java | 2 +- .../rest/controller/ProjectController.java | 2 +- .../kylinolap/rest/security/AclEntityFactory.java | 2 +- .../com/kylinolap/rest/service/BasicService.java | 2 +- .../com/kylinolap/rest/service/CubeService.java | 2 +- .../rest/controller/ProjectControllerTest.java | 2 +- .../rest/controller/QueryControllerTest.java | 2 +- .../kylinolap/rest/service/CubeServiceTest.java | 2 +- .../com/kylinolap/rest/service/JobServiceTest.java | 2 +- .../kylinolap/rest/service/QueryServiceTest.java | 2 +- 20 files changed, 31 insertions(+), 36 deletions(-) diff --git a/cube/src/main/java/com/kylinolap/cube/CubeManager.java b/cube/src/main/java/com/kylinolap/cube/CubeManager.java index 1813e7b..088091f 100644 --- a/cube/src/main/java/com/kylinolap/cube/CubeManager.java +++ b/cube/src/main/java/com/kylinolap/cube/CubeManager.java @@ -27,6 +27,7 @@ import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; +import com.kylinolap.metadata.project.ProjectInstance; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -40,7 +41,6 @@ import com.kylinolap.cube.exception.CubeIntegrityException; import com.kylinolap.cube.model.CubeDesc; import com.kylinolap.cube.model.DimensionDesc; -import com.kylinolap.cube.project.ProjectInstance; import com.kylinolap.cube.project.ProjectManager; import com.kylinolap.dict.DateStrDictionary; import com.kylinolap.dict.Dictionary; diff --git a/cube/src/test/java/com/kylinolap/cube/CubeManagerTest.java b/cube/src/test/java/com/kylinolap/cube/CubeManagerTest.java index 990ea37..b9e778c 100644 --- a/cube/src/test/java/com/kylinolap/cube/CubeManagerTest.java +++ b/cube/src/test/java/com/kylinolap/cube/CubeManagerTest.java @@ -18,6 +18,7 @@ import static org.junit.Assert.*; +import com.kylinolap.metadata.project.ProjectInstance; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -26,7 +27,6 @@ import com.kylinolap.common.util.JsonUtil; import com.kylinolap.common.util.LocalFileMetadataTestCase; import com.kylinolap.cube.model.CubeDesc; -import com.kylinolap.cube.project.ProjectInstance; import com.kylinolap.cube.project.ProjectManager; import com.kylinolap.metadata.MetadataManager; diff --git a/cube/src/test/java/com/kylinolap/cube/SegmentManagementTest.java b/cube/src/test/java/com/kylinolap/cube/SegmentManagementTest.java index 5d0558e..298e45f 100644 --- a/cube/src/test/java/com/kylinolap/cube/SegmentManagementTest.java +++ b/cube/src/test/java/com/kylinolap/cube/SegmentManagementTest.java @@ -24,6 +24,7 @@ import java.util.List; import java.util.TimeZone; +import com.kylinolap.metadata.project.ProjectInstance; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -33,7 +34,6 @@ import com.kylinolap.common.util.LocalFileMetadataTestCase; import com.kylinolap.cube.exception.CubeIntegrityException; import com.kylinolap.cube.model.CubeDesc; -import com.kylinolap.cube.project.ProjectInstance; import com.kylinolap.cube.project.ProjectManager; import com.kylinolap.metadata.MetadataManager; diff --git a/cube/src/test/java/com/kylinolap/cube/project/ProjectManagerTest.java b/cube/src/test/java/com/kylinolap/cube/project/ProjectManagerTest.java index eb03c56..b7fadfa 100644 --- a/cube/src/test/java/com/kylinolap/cube/project/ProjectManagerTest.java +++ b/cube/src/test/java/com/kylinolap/cube/project/ProjectManagerTest.java @@ -20,6 +20,7 @@ import java.io.IOException; +import com.kylinolap.metadata.project.ProjectInstance; import org.junit.After; import org.junit.Before; import org.junit.Test; diff --git a/job/src/main/java/com/kylinolap/job/JobManager.java b/job/src/main/java/com/kylinolap/job/JobManager.java index ec82f3a..8b94693 100644 --- a/job/src/main/java/com/kylinolap/job/JobManager.java +++ b/job/src/main/java/com/kylinolap/job/JobManager.java @@ -16,26 +16,10 @@ package com.kylinolap.job; -import java.io.IOException; -import java.net.UnknownHostException; -import java.text.SimpleDateFormat; -import java.util.ArrayList; -import java.util.Date; -import java.util.List; -import java.util.TimeZone; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import com.kylinolap.common.KylinConfig; -import com.kylinolap.cube.CubeBuildTypeEnum; -import com.kylinolap.cube.CubeInstance; -import com.kylinolap.cube.CubeManager; -import com.kylinolap.cube.CubeSegment; -import com.kylinolap.cube.CubeSegmentStatusEnum; +import com.kylinolap.cube.*; import com.kylinolap.cube.exception.CubeIntegrityException; import com.kylinolap.cube.model.CubeDesc; -import com.kylinolap.cube.project.ProjectInstance; import com.kylinolap.cube.project.ProjectManager; import com.kylinolap.job.JobInstance.JobStep; import com.kylinolap.job.constant.JobConstants; @@ -46,6 +30,17 @@ import com.kylinolap.job.exception.InvalidJobInstanceException; import com.kylinolap.job.exception.JobException; import com.kylinolap.job.hadoop.hive.JoinedFlatTableDesc; +import com.kylinolap.metadata.project.ProjectInstance; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.net.UnknownHostException; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.TimeZone; /** * @author xjiang, ysong1 @@ -173,7 +168,6 @@ public void discardJob(String uuid) throws IOException, CubeIntegrityException, } /** - * @param uuid * @param jobInstance * @throws IOException * @throws JobException diff --git a/job/src/main/java/com/kylinolap/job/tools/CubeMigrationCLI.java b/job/src/main/java/com/kylinolap/job/tools/CubeMigrationCLI.java index 6323fbb..831f3cd 100644 --- a/job/src/main/java/com/kylinolap/job/tools/CubeMigrationCLI.java +++ b/job/src/main/java/com/kylinolap/job/tools/CubeMigrationCLI.java @@ -6,14 +6,13 @@ import com.kylinolap.common.persistence.Serializer; import com.kylinolap.cube.*; import com.kylinolap.cube.model.CubeDesc; -import com.kylinolap.cube.project.ProjectInstance; import com.kylinolap.dict.DictionaryInfo; import com.kylinolap.dict.DictionaryManager; import com.kylinolap.dict.lookup.SnapshotManager; import com.kylinolap.dict.lookup.SnapshotTable; import com.kylinolap.job.JobInstance; import com.kylinolap.metadata.model.TableDesc; - +import com.kylinolap.metadata.project.ProjectInstance; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; diff --git a/metadata/src/main/java/com/kylinolap/metadata/project/ProjectInstance.java b/metadata/src/main/java/com/kylinolap/metadata/project/ProjectInstance.java index 00c5041..bb0ae33 100644 --- a/metadata/src/main/java/com/kylinolap/metadata/project/ProjectInstance.java +++ b/metadata/src/main/java/com/kylinolap/metadata/project/ProjectInstance.java @@ -267,4 +267,5 @@ public boolean apply(@Nullable ProjectDataModel input) { public String toString() { return "ProjectDesc [name=" + name + "]"; } + } diff --git a/query/src/main/java/com/kylinolap/query/schema/OLAPSchema.java b/query/src/main/java/com/kylinolap/query/schema/OLAPSchema.java index eda7244..da45eb7 100644 --- a/query/src/main/java/com/kylinolap/query/schema/OLAPSchema.java +++ b/query/src/main/java/com/kylinolap/query/schema/OLAPSchema.java @@ -19,12 +19,12 @@ import java.util.List; import java.util.Map; +import com.kylinolap.metadata.project.ProjectInstance; import net.hydromatic.optiq.Table; import net.hydromatic.optiq.impl.AbstractSchema; import com.kylinolap.common.KylinConfig; import com.kylinolap.cube.CubeManager; -import com.kylinolap.cube.project.ProjectInstance; import com.kylinolap.cube.project.ProjectManager; import com.kylinolap.metadata.MetadataManager; import com.kylinolap.metadata.model.TableDesc; diff --git a/query/src/main/java/com/kylinolap/query/schema/OLAPSchemaFactory.java b/query/src/main/java/com/kylinolap/query/schema/OLAPSchemaFactory.java index 18b5d5c..dd5fc7f 100644 --- a/query/src/main/java/com/kylinolap/query/schema/OLAPSchemaFactory.java +++ b/query/src/main/java/com/kylinolap/query/schema/OLAPSchemaFactory.java @@ -22,6 +22,7 @@ import java.util.List; import java.util.Map; +import com.kylinolap.metadata.project.ProjectInstance; import net.hydromatic.optiq.Schema; import net.hydromatic.optiq.SchemaFactory; import net.hydromatic.optiq.SchemaPlus; @@ -29,7 +30,6 @@ import org.eigenbase.util14.ConversionUtil; import com.kylinolap.common.KylinConfig; -import com.kylinolap.cube.project.ProjectInstance; import com.kylinolap.cube.project.ProjectManager; import com.kylinolap.metadata.model.DatabaseDesc; import com.kylinolap.metadata.model.TableDesc; diff --git a/query/src/test/java/com/kylinolap/query/test/KylinQueryTest.java b/query/src/test/java/com/kylinolap/query/test/KylinQueryTest.java index 0df1599..30c9403 100644 --- a/query/src/test/java/com/kylinolap/query/test/KylinQueryTest.java +++ b/query/src/test/java/com/kylinolap/query/test/KylinQueryTest.java @@ -35,7 +35,7 @@ import com.kylinolap.common.KylinConfig; import com.kylinolap.common.util.HBaseMetadataTestCase; import com.kylinolap.cube.CubeManager; -import com.kylinolap.cube.project.ProjectInstance; +import com.kylinolap.metadata.project.ProjectInstance; import com.kylinolap.query.enumerator.OLAPQuery; import com.kylinolap.query.relnode.OLAPContext; import com.kylinolap.query.schema.OLAPSchemaFactory; diff --git a/server/src/main/java/com/kylinolap/rest/controller/CubeController.java b/server/src/main/java/com/kylinolap/rest/controller/CubeController.java index c7cb2d4..1e208b0 100644 --- a/server/src/main/java/com/kylinolap/rest/controller/CubeController.java +++ b/server/src/main/java/com/kylinolap/rest/controller/CubeController.java @@ -23,6 +23,7 @@ import java.util.Map; import java.util.UUID; +import com.kylinolap.metadata.project.ProjectInstance; import com.kylinolap.storage.hbase.coprocessor.observer.ObserverEnabler; import org.apache.commons.lang.StringUtils; @@ -48,7 +49,6 @@ import com.kylinolap.cube.CubeSegment; import com.kylinolap.cube.exception.CubeIntegrityException; import com.kylinolap.cube.model.CubeDesc; -import com.kylinolap.cube.project.ProjectInstance; import com.kylinolap.job.JobInstance; import com.kylinolap.job.exception.InvalidJobInstanceException; import com.kylinolap.job.exception.JobException; diff --git a/server/src/main/java/com/kylinolap/rest/controller/ProjectController.java b/server/src/main/java/com/kylinolap/rest/controller/ProjectController.java index 3b15dc9..2fc9950 100644 --- a/server/src/main/java/com/kylinolap/rest/controller/ProjectController.java +++ b/server/src/main/java/com/kylinolap/rest/controller/ProjectController.java @@ -19,6 +19,7 @@ import java.io.IOException; import java.util.List; +import com.kylinolap.metadata.project.ProjectInstance; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -32,7 +33,6 @@ import org.springframework.web.bind.annotation.ResponseBody; import com.codahale.metrics.annotation.Metered; -import com.kylinolap.cube.project.ProjectInstance; import com.kylinolap.rest.exception.InternalErrorException; import com.kylinolap.rest.request.CreateProjectRequest; import com.kylinolap.rest.request.UpdateProjectRequest; diff --git a/server/src/main/java/com/kylinolap/rest/security/AclEntityFactory.java b/server/src/main/java/com/kylinolap/rest/security/AclEntityFactory.java index cb51a0a..587dbcd 100644 --- a/server/src/main/java/com/kylinolap/rest/security/AclEntityFactory.java +++ b/server/src/main/java/com/kylinolap/rest/security/AclEntityFactory.java @@ -18,8 +18,8 @@ import com.kylinolap.common.persistence.RootPersistentEntity; import com.kylinolap.cube.CubeInstance; -import com.kylinolap.cube.project.ProjectInstance; import com.kylinolap.job.JobInstance; +import com.kylinolap.metadata.project.ProjectInstance; /** * @author xduo diff --git a/server/src/main/java/com/kylinolap/rest/service/BasicService.java b/server/src/main/java/com/kylinolap/rest/service/BasicService.java index d202313..87f5eae 100644 --- a/server/src/main/java/com/kylinolap/rest/service/BasicService.java +++ b/server/src/main/java/com/kylinolap/rest/service/BasicService.java @@ -31,6 +31,7 @@ import javax.sql.DataSource; +import com.kylinolap.metadata.project.ProjectInstance; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -42,7 +43,6 @@ import com.kylinolap.common.KylinConfig; import com.kylinolap.cube.CubeDescManager; import com.kylinolap.cube.CubeManager; -import com.kylinolap.cube.project.ProjectInstance; import com.kylinolap.cube.project.ProjectManager; import com.kylinolap.job.JobManager; import com.kylinolap.job.engine.JobEngineConfig; diff --git a/server/src/main/java/com/kylinolap/rest/service/CubeService.java b/server/src/main/java/com/kylinolap/rest/service/CubeService.java index 4c3eea9..de58372 100644 --- a/server/src/main/java/com/kylinolap/rest/service/CubeService.java +++ b/server/src/main/java/com/kylinolap/rest/service/CubeService.java @@ -31,6 +31,7 @@ import java.util.Map; import java.util.Set; +import com.kylinolap.metadata.project.ProjectInstance; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; @@ -61,7 +62,6 @@ import com.kylinolap.cube.cuboid.CuboidCLI; import com.kylinolap.cube.exception.CubeIntegrityException; import com.kylinolap.cube.model.CubeDesc; -import com.kylinolap.cube.project.ProjectInstance; import com.kylinolap.job.JobDAO; import com.kylinolap.job.JobInstance; import com.kylinolap.job.JobInstance.JobStep; diff --git a/server/src/test/java/com/kylinolap/rest/controller/ProjectControllerTest.java b/server/src/test/java/com/kylinolap/rest/controller/ProjectControllerTest.java index af52415..de25bc9 100644 --- a/server/src/test/java/com/kylinolap/rest/controller/ProjectControllerTest.java +++ b/server/src/test/java/com/kylinolap/rest/controller/ProjectControllerTest.java @@ -8,7 +8,7 @@ import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; -import com.kylinolap.cube.project.ProjectInstance; +import com.kylinolap.metadata.project.ProjectInstance; import com.kylinolap.cube.project.ProjectManager; import com.kylinolap.rest.exception.InternalErrorException; import com.kylinolap.rest.request.CreateProjectRequest; diff --git a/server/src/test/java/com/kylinolap/rest/controller/QueryControllerTest.java b/server/src/test/java/com/kylinolap/rest/controller/QueryControllerTest.java index 922e4d9..d1c6f32 100644 --- a/server/src/test/java/com/kylinolap/rest/controller/QueryControllerTest.java +++ b/server/src/test/java/com/kylinolap/rest/controller/QueryControllerTest.java @@ -22,7 +22,7 @@ import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; -import com.kylinolap.cube.project.ProjectInstance; +import com.kylinolap.metadata.project.ProjectInstance; import com.kylinolap.rest.request.MetaRequest; import com.kylinolap.rest.request.SQLRequest; import com.kylinolap.rest.service.QueryService; diff --git a/server/src/test/java/com/kylinolap/rest/service/CubeServiceTest.java b/server/src/test/java/com/kylinolap/rest/service/CubeServiceTest.java index 24c86be..c228f17 100644 --- a/server/src/test/java/com/kylinolap/rest/service/CubeServiceTest.java +++ b/server/src/test/java/com/kylinolap/rest/service/CubeServiceTest.java @@ -25,7 +25,7 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.kylinolap.cube.CubeInstance; -import com.kylinolap.cube.project.ProjectInstance; +import com.kylinolap.metadata.project.ProjectInstance; import com.kylinolap.job.exception.JobException; /** diff --git a/server/src/test/java/com/kylinolap/rest/service/JobServiceTest.java b/server/src/test/java/com/kylinolap/rest/service/JobServiceTest.java index 6d63dd4..d0c615a 100644 --- a/server/src/test/java/com/kylinolap/rest/service/JobServiceTest.java +++ b/server/src/test/java/com/kylinolap/rest/service/JobServiceTest.java @@ -22,7 +22,7 @@ import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; -import com.kylinolap.cube.project.ProjectInstance; +import com.kylinolap.metadata.project.ProjectInstance; import com.kylinolap.job.exception.JobException; /** diff --git a/server/src/test/java/com/kylinolap/rest/service/QueryServiceTest.java b/server/src/test/java/com/kylinolap/rest/service/QueryServiceTest.java index 157cdae..f61ca31 100644 --- a/server/src/test/java/com/kylinolap/rest/service/QueryServiceTest.java +++ b/server/src/test/java/com/kylinolap/rest/service/QueryServiceTest.java @@ -24,7 +24,7 @@ import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; -import com.kylinolap.cube.project.ProjectInstance; +import com.kylinolap.metadata.project.ProjectInstance; import com.kylinolap.job.exception.JobException; import com.kylinolap.rest.request.SQLRequest; import com.kylinolap.rest.response.SQLResponse; From 2a23feb9c6b09a47cf0f3d9e5b3df2e318826885 Mon Sep 17 00:00:00 2001 From: "qianhao.zhou" Date: Mon, 8 Dec 2014 17:20:38 +0800 Subject: [PATCH 3/4] rename ProjectManager --- .../main/java/com/kylinolap/cube/CubeManager.java | 2 +- .../cube/project/CubeRealizationManager.java | 584 +++++++++++++++++++++ .../com/kylinolap/cube/project/ProjectManager.java | 584 --------------------- .../main/java/com/kylinolap/job/JobManager.java | 2 +- .../kylinolap/metadata/project/ProjectManager.java | 12 + .../com/kylinolap/query/routing/QueryRouter.java | 14 +- .../com/kylinolap/query/schema/OLAPSchema.java | 7 +- .../kylinolap/query/schema/OLAPSchemaFactory.java | 4 +- .../java/com/kylinolap/query/schema/OLAPTable.java | 4 +- .../kylinolap/rest/controller/TableController.java | 2 +- .../com/kylinolap/rest/service/BasicService.java | 10 +- 11 files changed, 616 insertions(+), 609 deletions(-) create mode 100644 cube/src/main/java/com/kylinolap/cube/project/CubeRealizationManager.java delete mode 100644 cube/src/main/java/com/kylinolap/cube/project/ProjectManager.java diff --git a/cube/src/main/java/com/kylinolap/cube/CubeManager.java b/cube/src/main/java/com/kylinolap/cube/CubeManager.java index 088091f..1302c5d 100644 --- a/cube/src/main/java/com/kylinolap/cube/CubeManager.java +++ b/cube/src/main/java/com/kylinolap/cube/CubeManager.java @@ -41,7 +41,7 @@ import com.kylinolap.cube.exception.CubeIntegrityException; import com.kylinolap.cube.model.CubeDesc; import com.kylinolap.cube.model.DimensionDesc; -import com.kylinolap.cube.project.ProjectManager; +import com.kylinolap.metadata.project.ProjectManager; import com.kylinolap.dict.DateStrDictionary; import com.kylinolap.dict.Dictionary; import com.kylinolap.dict.DictionaryInfo; diff --git a/cube/src/main/java/com/kylinolap/cube/project/CubeRealizationManager.java b/cube/src/main/java/com/kylinolap/cube/project/CubeRealizationManager.java new file mode 100644 index 0000000..956000f --- /dev/null +++ b/cube/src/main/java/com/kylinolap/cube/project/CubeRealizationManager.java @@ -0,0 +1,584 @@ +/* + * Copyright 2013-2014 eBay Software Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kylinolap.cube.project; + +import java.io.IOException; +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; + +import com.kylinolap.metadata.project.ProjectInstance; +import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.collect.HashMultimap; +import com.google.common.collect.Lists; +import com.google.common.collect.Multimap; +import com.google.common.collect.Multimaps; +import com.kylinolap.common.KylinConfig; +import com.kylinolap.common.persistence.JsonSerializer; +import com.kylinolap.common.persistence.ResourceStore; +import com.kylinolap.common.persistence.Serializer; +import com.kylinolap.common.restclient.Broadcaster; +import com.kylinolap.common.restclient.SingleValueCache; +import com.kylinolap.cube.CubeInstance; +import com.kylinolap.cube.CubeManager; +import com.kylinolap.cube.model.CubeDesc; +import com.kylinolap.cube.model.DimensionDesc; +import com.kylinolap.cube.model.MeasureDesc; +import com.kylinolap.metadata.MetadataManager; +import com.kylinolap.metadata.model.ColumnDesc; +import com.kylinolap.metadata.model.JoinDesc; +import com.kylinolap.metadata.model.TableDesc; +import com.kylinolap.metadata.model.realization.FunctionDesc; +import com.kylinolap.metadata.model.realization.TblColRef; + +/** + * @author xduo + */ +public class CubeRealizationManager { + private static final Logger logger = LoggerFactory.getLogger(CubeRealizationManager.class); + + // static cached instances + private static final ConcurrentHashMap CACHE = new ConcurrentHashMap(); + private static final Serializer PROJECT_SERIALIZER = new JsonSerializer(ProjectInstance.class); + + private KylinConfig config; + // project name => ProjrectDesc + private SingleValueCache projectMap = new SingleValueCache(Broadcaster.TYPE.PROJECT); + // project name => tables + private Multimap projectTables = Multimaps.synchronizedMultimap(HashMultimap.create()); + + public static CubeRealizationManager getInstance(KylinConfig config) { + CubeRealizationManager r = CACHE.get(config); + if (r != null) { + return r; + } + + synchronized (CubeRealizationManager.class) { + r = CACHE.get(config); + if (r != null) { + return r; + } + try { + r = new CubeRealizationManager(config); + CACHE.put(config, r); + if (CACHE.size() > 1) { + logger.warn("More than one singleton exist"); + } + return r; + } catch (IOException e) { + throw new IllegalStateException("Failed to init CubeManager from " + config, e); + } + } + } + + public static synchronized void removeInstance(KylinConfig config) { + CACHE.remove(config); + } + + private CubeRealizationManager(KylinConfig config) throws IOException { + logger.info("Initializing CubeManager with metadata url " + config); + this.config = config; + + loadAllProjects(); + } + + public static String getDefaultProjectName() { + return ProjectInstance.DEFAULT_PROJECT_NAME; + } + + public List listAllProjects() { + return new ArrayList(projectMap.values()); + } + + public List getProjects(String cubeName) { + return this.findProjects(cubeName); + } + + public ProjectInstance dropProject(String projectName) throws IOException { + if (projectName == null) + throw new IllegalArgumentException("Project name not given"); + + ProjectInstance projectInstance = getProject(projectName); + + if (projectInstance == null) { + throw new IllegalStateException("The project named " + projectName + " does not exist"); + } + + if (projectInstance.getCubes().size() != 0) { + throw new IllegalStateException("The project named " + projectName + " can not be deleted because there's still cubes in it. Delete all the cubes first."); + } + + logger.info("Dropping project '" + projectInstance.getName() + "'"); + deleteResource(projectInstance); + + return projectInstance; + } + + public ProjectInstance getProject(String projectName) { + if (projectName == null) + return null; + projectName = ProjectInstance.getNormalizedProjectName(projectName); + return projectMap.get(projectName); + } + + public ProjectInstance createProject(String projectName, String owner, String description) throws IOException { + + logger.info("Creating project '" + projectName); + + ProjectInstance currentProject = getProject(projectName); + if (currentProject == null) { + currentProject = ProjectInstance.create(projectName, owner, description, null); + } else { + throw new IllegalStateException("The project named " + projectName + "already exists"); + } + + saveResource(currentProject); + + return currentProject; + } + + public ProjectInstance updateProject(ProjectInstance project, String newName, String newDesc) throws IOException { + if (!project.getName().equals(newName)) { + ProjectInstance newProject = this.createProject(newName, project.getOwner(), newDesc); + newProject.setCreateTime(project.getCreateTime()); + newProject.recordUpdateTime(System.currentTimeMillis()); + newProject.setCubes(project.getCubes()); + + deleteResource(project); + saveResource(newProject); + + return newProject; + } else { + project.setName(newName); + project.setDescription(newDesc); + + if (project.getUuid() == null) + project.updateRandomUuid(); + + saveResource(project); + + return project; + } + } + + public boolean isCubeInProject(String projectName, CubeInstance cube) { + return this.listAllCubes(projectName).contains(cube); + } + + public ProjectInstance updateCubeToProject(String cubeName, String newProjectName, String owner) throws IOException { + removeCubeFromProjects(cubeName); + + return addCubeToProject(cubeName, newProjectName, owner); + } + + public ProjectInstance updateTableToProject(String tables, String projectName) throws IOException { + ProjectInstance projectInstance = getProject(projectName); + String[] tokens = StringUtils.split(tables, ","); + for (int i = 0; i < tokens.length; i++) { + String token = tokens[i].trim(); + if (StringUtils.isNotEmpty(token)) { + projectInstance.addTable(token); + } + } + + List exposedTables = listExposedTables(projectName); + for (TableDesc table : exposedTables) { + projectInstance.addTable(table.getIdentity()); + } + + saveResource(projectInstance); + return projectInstance; + } + + + public void removeCubeFromProjects(String cubeName) throws IOException { + for (ProjectInstance projectInstance : findProjects(cubeName)) { + projectInstance.removeCube(cubeName); + + saveResource(projectInstance); + } + } + + public List listExposedTables(String project) { + project = ProjectInstance.getNormalizedProjectName(project); + List tables = Lists.newArrayList(); + + for (ProjectTable table : projectTables.get(project)) { + TableDesc tableDesc = getMetadataManager().getTableDesc(table.getName()); + if (tableDesc != null) { + tables.add(tableDesc); + } + } + + return tables; + } + + + public List listDefinedTablesInProject(String project) throws IOException { + if(null==project){ + return Collections.emptyList(); + } + project = ProjectInstance.getNormalizedProjectName(project); + ProjectInstance projectInstance = getProject(project); + int originTableCount = projectInstance.getTablesCount(); + //sync exposed table to project when list + List exposedTables = listExposedTables(project); + for (TableDesc table : exposedTables) { + projectInstance.addTable(TableDesc.getTableIdentity(table)); + } + //only save project json if new tables are sync in + if (originTableCount < projectInstance.getTablesCount()) { + saveResource(projectInstance); + } + + List tables = Lists.newArrayList(); + for (String table : projectInstance.getTables()) { + TableDesc tableDesc = getMetadataManager().getTableDesc(table); + if (tableDesc != null) { + tables.add(tableDesc); + } + } + + return tables; + } + + public List listExposedColumns(String project, String table) { + project = ProjectInstance.getNormalizedProjectName(project); + + MetadataManager metaMgr = getMetadataManager(); + TableDesc tableDesc = metaMgr.getTableDesc(table); + List columns = Lists.newArrayList(); + + for (String column : this.getProjectTable(project, table).getColumns()) { + columns.add(tableDesc.findColumnByName(column)); + } + + return columns; + } + + public boolean isExposedTable(String project, String table) { + project = ProjectInstance.getNormalizedProjectName(project); + + return projectTables.containsEntry(project, new ProjectTable(table)); + } + + public boolean isExposedColumn(String project, String table, String col) { + project = ProjectInstance.getNormalizedProjectName(project); + + return getProjectTable(project, table).getColumns().contains(col); + } + + public List listAllCubes(String project) { + project = ProjectInstance.getNormalizedProjectName(project); + + HashSet ret = new HashSet(); + + ProjectInstance projectInstance = getProject(project); + if (projectInstance != null) { + for (String cubeName : projectInstance.getCubes()) { + CubeInstance cube = CubeManager.getInstance(config).getCube(cubeName); + if (null != cube) { + ret.add(cube); + } else { + logger.error("Failed to load cube " + cubeName); + } + } + } + + return new ArrayList(ret); + } + + + public List getCubesByTable(String project, String tableName) { + project = ProjectInstance.getNormalizedProjectName(project); + tableName = tableName.toUpperCase(); + List cubes = new ArrayList(); + + ProjectTable projectTable = getProjectTable(project, tableName); + cubes.addAll(projectTable.getCubes()); + + return cubes; + } + + public List getOnlineCubesByFactTable(String project, String factTableName) { + project = ProjectInstance.getNormalizedProjectName(project); + factTableName = factTableName.toUpperCase(); + List cubes = new ArrayList(); + ProjectTable projectTable = this.getProjectTable(project, factTableName); + for (CubeInstance cube : projectTable.getCubes()) { + if (cube.getDescriptor().getModel().isFactTable(factTableName) && cube.isReady()) { + cubes.add(cube); + } + } + + return cubes; + } + + public List listEffectiveRewriteMeasures(String project, String factTable) { + factTable = factTable.toUpperCase(); + + HashSet relatedDesc = new HashSet(); + for (CubeInstance cube : getProjectTable(project, factTable).getCubes()) { + if (cube.isReady() == false) + continue; + if (cube.getDescriptor().getModel().isFactTable(factTable) == false) + continue; + + relatedDesc.add(cube.getDescriptor()); + } + + List result = Lists.newArrayList(); + for (CubeDesc desc : relatedDesc) { + for (MeasureDesc m : desc.getMeasures()) { + FunctionDesc func = m.getFunction(); + if (func.needRewrite()) + result.add(m); + } + } + + return result; + } + + public void loadProjectCache(ProjectInstance project, boolean triggerUpdate) throws IOException { + loadProject(project.getResourcePath(), triggerUpdate); + loadTables(project.getResourcePath()); + } + + public void removeProjectCache(ProjectInstance project) { + String projectName = ProjectInstance.getNormalizedProjectName(project.getName()); + if (projectMap.containsKey(projectName)) { + projectMap.remove(projectName); + projectTables.removeAll(projectName); + } + } + + private void mapTableToCube(ProjectInstance projectInstance, CubeInstance cubeInstance) { + // schema sanity check + CubeDesc cubeDesc = cubeInstance.getDescriptor(); + if (cubeDesc == null) { + logger.warn("No CubeDesc found by name '" + cubeInstance.getDescName() + "'"); + return; + } + + // table ==> cube mapping + String factTable = cubeDesc.getFactTable(); + logger.debug("Fact Table: " + factTable + " -- Cube: " + cubeInstance.getName()); + assert this.getMetadataManager().getTableDesc(factTable) != null; + + String project = ProjectInstance.getNormalizedProjectName(projectInstance.getName()); + ProjectTable factProjTable = this.getProjectTable(project, factTable, true); + if (!factProjTable.getCubes().contains(cubeInstance)) { + factProjTable.getCubes().add(cubeInstance); + } + + for (DimensionDesc d : cubeDesc.getDimensions()) { + String lookupTable = d.getTable(); + logger.debug("Lookup Table: " + lookupTable + " -- Cube: " + cubeInstance.getName()); + assert this.getMetadataManager().getTableDesc(lookupTable) != null; + + ProjectTable dimensionProjTable = this.getProjectTable(project, lookupTable); + if (!dimensionProjTable.getCubes().contains(cubeInstance)) { + dimensionProjTable.getCubes().add(cubeInstance); + } + } + } + + private List findProjects(String cubeName) { + List projects = new ArrayList(); + for (ProjectInstance projectInstance : projectMap.values()) { + if (projectInstance.containsCube(cubeName)) { + projects.add(projectInstance); + } + } + + return projects; + } + + private synchronized ProjectInstance loadProject(String path, boolean triggerUpdate) throws IOException { + ResourceStore store = getStore(); + logger.debug("Loading CubeInstance " + store.getReadableResourcePath(path)); + + ProjectInstance projectInstance = store.getResource(path, ProjectInstance.class, PROJECT_SERIALIZER); + projectInstance.init(); + + if (StringUtils.isBlank(projectInstance.getName())) { + throw new IllegalStateException("Project name must not be blank"); + } + + if (triggerUpdate) { + projectMap.put(projectInstance.getName().toUpperCase(), projectInstance); + } else { + projectMap.putLocal(projectInstance.getName().toUpperCase(), projectInstance); + } + + return projectInstance; + } + + private synchronized void loadTables(String path) throws IOException { + ResourceStore store = getStore(); + logger.debug("Loading CubeInstance " + store.getReadableResourcePath(path)); + + ProjectInstance projectInstance = store.getResource(path, ProjectInstance.class, PROJECT_SERIALIZER); + projectInstance.init(); + + String project = ProjectInstance.getNormalizedProjectName(projectInstance.getName()); + projectTables.removeAll(project); + + for (CubeInstance cubeInstance : this.listAllCubes(projectInstance.getName())) { + markExposedTablesAndColumns(projectInstance.getName(), cubeInstance); + mapTableToCube(projectInstance, cubeInstance); + } + } + + private void loadAllProjects() throws IOException { + ResourceStore store = getStore(); + List paths = store.collectResourceRecursively(ResourceStore.PROJECT_RESOURCE_ROOT, ".json"); + + logger.debug("Loading Project from folder " + store.getReadableResourcePath(ResourceStore.PROJECT_RESOURCE_ROOT)); + + for (String path : paths) { + loadProject(path, false); + loadTables(path); + } + + logger.debug("Loaded " + paths.size() + " Project(s)"); + } + + private ProjectInstance addCubeToProject(String cubeName, String project, String user) throws IOException { + String newProjectName = ProjectInstance.getNormalizedProjectName(project); + ProjectInstance newProject = getProject(newProjectName); + if (newProject == null) { + newProject = this.createProject(newProjectName, user, "This is a project automatically added when adding cube " + cubeName); + } + newProject.addCube(cubeName); + saveResource(newProject); + + return newProject; + } + + private void saveResource(ProjectInstance proj) throws IOException { + ResourceStore store = getStore(); + store.putResource(proj.getResourcePath(), proj, PROJECT_SERIALIZER); + afterProjectUpdated(proj); + } + + private void deleteResource(ProjectInstance proj) throws IOException { + ResourceStore store = getStore(); + store.deleteResource(proj.getResourcePath()); + this.afterProjectDropped(proj); + } + + private void afterProjectUpdated(ProjectInstance updatedProject) { + try { + this.loadProjectCache(updatedProject, true); + } catch (IOException e) { + logger.error(e.getLocalizedMessage(), e); + } + } + + private void afterProjectDropped(ProjectInstance droppedProject) { + this.removeProjectCache(droppedProject); + } + + // sync on update + private void markExposedTablesAndColumns(String projectName, CubeInstance cubeInstance) { + if (!cubeInstance.isReady()) + return; + + CubeDesc cubeDesc = cubeInstance.getDescriptor(); + String factTable = cubeDesc.getFactTable(); + for (DimensionDesc dim : cubeDesc.getDimensions()) { + String lookupTable = dim.getTable(); + JoinDesc join = dim.getJoin(); + if (join == null) + continue; // for dimensions on fact table, there's no join + + if (join.getForeignKeyColumns() == null) { + throw new IllegalStateException("Null FK for " + join); + } + for (TblColRef fkCol : join.getForeignKeyColumns()) { + markExposedTableAndColumn(projectName, factTable, fkCol.getName(), dim); + } + if (join.getPrimaryKeyColumns() == null) { + throw new IllegalStateException("Null PK for " + join); + } + for (TblColRef pkCol : join.getPrimaryKeyColumns()) { + markExposedTableAndColumn(projectName, lookupTable, pkCol.getName(), dim); + } + } + for (TblColRef col : cubeDesc.listAllColumns()) { + markExposedTableAndColumn(projectName, col.getTable(), col.getName(), col); + } + } + + private void markExposedTableAndColumn(String project, String table, String column, Object refObj) { + project = ProjectInstance.getNormalizedProjectName(project); + TableDesc t = this.getMetadataManager().getTableDesc(table); + if (t == null) { + throw new IllegalStateException("No SourceTable found by name '" + table + "', ref by " + refObj); + } + table = t.getIdentity(); // ensures upper case + + ProjectTable projTable = getProjectTable(project, table, true); + + ColumnDesc srcCol = t.findColumnByName(column); + if (srcCol == null) { + throw new IllegalStateException("No SourceColumn found by name '" + table + "/" + column + "', ref by " + refObj); + } + + if (!projTable.getColumns().contains(srcCol.getName())) { + projTable.getColumns().add(srcCol.getName()); + } + } + + private ProjectTable getProjectTable(String project, final String table) { + return getProjectTable(project, table, false); + } + + private ProjectTable getProjectTable(String project, final String table, boolean autoCreate) { + ProjectTable projectTable = null; + project = ProjectInstance.getNormalizedProjectName(project); + + if (this.projectTables.containsEntry(project, new ProjectTable(table))) { + Collection projects = this.projectTables.get(project); + for (ProjectTable pt : projects) { + if (pt.getName().equalsIgnoreCase(table)) { + projectTable = pt; + break; + } + } + } else { + projectTable = new ProjectTable(table); + + if (autoCreate) { + this.projectTables.put(project, projectTable); + } + } + + return projectTable; + } + + private ResourceStore getStore() { + return ResourceStore.getStore(this.config); + } + + private MetadataManager getMetadataManager() { + return MetadataManager.getInstance(config); + } +} diff --git a/cube/src/main/java/com/kylinolap/cube/project/ProjectManager.java b/cube/src/main/java/com/kylinolap/cube/project/ProjectManager.java deleted file mode 100644 index ccdd31d..0000000 --- a/cube/src/main/java/com/kylinolap/cube/project/ProjectManager.java +++ /dev/null @@ -1,584 +0,0 @@ -/* - * Copyright 2013-2014 eBay Software Foundation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.kylinolap.cube.project; - -import java.io.IOException; -import java.util.*; -import java.util.concurrent.ConcurrentHashMap; - -import com.kylinolap.metadata.project.ProjectInstance; -import org.apache.commons.lang3.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.google.common.collect.HashMultimap; -import com.google.common.collect.Lists; -import com.google.common.collect.Multimap; -import com.google.common.collect.Multimaps; -import com.kylinolap.common.KylinConfig; -import com.kylinolap.common.persistence.JsonSerializer; -import com.kylinolap.common.persistence.ResourceStore; -import com.kylinolap.common.persistence.Serializer; -import com.kylinolap.common.restclient.Broadcaster; -import com.kylinolap.common.restclient.SingleValueCache; -import com.kylinolap.cube.CubeInstance; -import com.kylinolap.cube.CubeManager; -import com.kylinolap.cube.model.CubeDesc; -import com.kylinolap.cube.model.DimensionDesc; -import com.kylinolap.cube.model.MeasureDesc; -import com.kylinolap.metadata.MetadataManager; -import com.kylinolap.metadata.model.ColumnDesc; -import com.kylinolap.metadata.model.JoinDesc; -import com.kylinolap.metadata.model.TableDesc; -import com.kylinolap.metadata.model.realization.FunctionDesc; -import com.kylinolap.metadata.model.realization.TblColRef; - -/** - * @author xduo - */ -public class ProjectManager { - private static final Logger logger = LoggerFactory.getLogger(ProjectManager.class); - - // static cached instances - private static final ConcurrentHashMap CACHE = new ConcurrentHashMap(); - private static final Serializer PROJECT_SERIALIZER = new JsonSerializer(ProjectInstance.class); - - private KylinConfig config; - // project name => ProjrectDesc - private SingleValueCache projectMap = new SingleValueCache(Broadcaster.TYPE.PROJECT); - // project name => tables - private Multimap projectTables = Multimaps.synchronizedMultimap(HashMultimap.create()); - - public static ProjectManager getInstance(KylinConfig config) { - ProjectManager r = CACHE.get(config); - if (r != null) { - return r; - } - - synchronized (ProjectManager.class) { - r = CACHE.get(config); - if (r != null) { - return r; - } - try { - r = new ProjectManager(config); - CACHE.put(config, r); - if (CACHE.size() > 1) { - logger.warn("More than one singleton exist"); - } - return r; - } catch (IOException e) { - throw new IllegalStateException("Failed to init CubeManager from " + config, e); - } - } - } - - public static synchronized void removeInstance(KylinConfig config) { - CACHE.remove(config); - } - - private ProjectManager(KylinConfig config) throws IOException { - logger.info("Initializing CubeManager with metadata url " + config); - this.config = config; - - loadAllProjects(); - } - - public static String getDefaultProjectName() { - return ProjectInstance.DEFAULT_PROJECT_NAME; - } - - public List listAllProjects() { - return new ArrayList(projectMap.values()); - } - - public List getProjects(String cubeName) { - return this.findProjects(cubeName); - } - - public ProjectInstance dropProject(String projectName) throws IOException { - if (projectName == null) - throw new IllegalArgumentException("Project name not given"); - - ProjectInstance projectInstance = getProject(projectName); - - if (projectInstance == null) { - throw new IllegalStateException("The project named " + projectName + " does not exist"); - } - - if (projectInstance.getCubes().size() != 0) { - throw new IllegalStateException("The project named " + projectName + " can not be deleted because there's still cubes in it. Delete all the cubes first."); - } - - logger.info("Dropping project '" + projectInstance.getName() + "'"); - deleteResource(projectInstance); - - return projectInstance; - } - - public ProjectInstance getProject(String projectName) { - if (projectName == null) - return null; - projectName = ProjectInstance.getNormalizedProjectName(projectName); - return projectMap.get(projectName); - } - - public ProjectInstance createProject(String projectName, String owner, String description) throws IOException { - - logger.info("Creating project '" + projectName); - - ProjectInstance currentProject = getProject(projectName); - if (currentProject == null) { - currentProject = ProjectInstance.create(projectName, owner, description, null); - } else { - throw new IllegalStateException("The project named " + projectName + "already exists"); - } - - saveResource(currentProject); - - return currentProject; - } - - public ProjectInstance updateProject(ProjectInstance project, String newName, String newDesc) throws IOException { - if (!project.getName().equals(newName)) { - ProjectInstance newProject = this.createProject(newName, project.getOwner(), newDesc); - newProject.setCreateTime(project.getCreateTime()); - newProject.recordUpdateTime(System.currentTimeMillis()); - newProject.setCubes(project.getCubes()); - - deleteResource(project); - saveResource(newProject); - - return newProject; - } else { - project.setName(newName); - project.setDescription(newDesc); - - if (project.getUuid() == null) - project.updateRandomUuid(); - - saveResource(project); - - return project; - } - } - - public boolean isCubeInProject(String projectName, CubeInstance cube) { - return this.listAllCubes(projectName).contains(cube); - } - - public ProjectInstance updateCubeToProject(String cubeName, String newProjectName, String owner) throws IOException { - removeCubeFromProjects(cubeName); - - return addCubeToProject(cubeName, newProjectName, owner); - } - - public ProjectInstance updateTableToProject(String tables, String projectName) throws IOException { - ProjectInstance projectInstance = getProject(projectName); - String[] tokens = StringUtils.split(tables, ","); - for (int i = 0; i < tokens.length; i++) { - String token = tokens[i].trim(); - if (StringUtils.isNotEmpty(token)) { - projectInstance.addTable(token); - } - } - - List exposedTables = listExposedTables(projectName); - for (TableDesc table : exposedTables) { - projectInstance.addTable(table.getIdentity()); - } - - saveResource(projectInstance); - return projectInstance; - } - - - public void removeCubeFromProjects(String cubeName) throws IOException { - for (ProjectInstance projectInstance : findProjects(cubeName)) { - projectInstance.removeCube(cubeName); - - saveResource(projectInstance); - } - } - - public List listExposedTables(String project) { - project = ProjectInstance.getNormalizedProjectName(project); - List tables = Lists.newArrayList(); - - for (ProjectTable table : projectTables.get(project)) { - TableDesc tableDesc = getMetadataManager().getTableDesc(table.getName()); - if (tableDesc != null) { - tables.add(tableDesc); - } - } - - return tables; - } - - - public List listDefinedTablesInProject(String project) throws IOException { - if(null==project){ - return Collections.emptyList(); - } - project = ProjectInstance.getNormalizedProjectName(project); - ProjectInstance projectInstance = getProject(project); - int originTableCount = projectInstance.getTablesCount(); - //sync exposed table to project when list - List exposedTables = listExposedTables(project); - for (TableDesc table : exposedTables) { - projectInstance.addTable(TableDesc.getTableIdentity(table)); - } - //only save project json if new tables are sync in - if (originTableCount < projectInstance.getTablesCount()) { - saveResource(projectInstance); - } - - List tables = Lists.newArrayList(); - for (String table : projectInstance.getTables()) { - TableDesc tableDesc = getMetadataManager().getTableDesc(table); - if (tableDesc != null) { - tables.add(tableDesc); - } - } - - return tables; - } - - public List listExposedColumns(String project, String table) { - project = ProjectInstance.getNormalizedProjectName(project); - - MetadataManager metaMgr = getMetadataManager(); - TableDesc tableDesc = metaMgr.getTableDesc(table); - List columns = Lists.newArrayList(); - - for (String column : this.getProjectTable(project, table).getColumns()) { - columns.add(tableDesc.findColumnByName(column)); - } - - return columns; - } - - public boolean isExposedTable(String project, String table) { - project = ProjectInstance.getNormalizedProjectName(project); - - return projectTables.containsEntry(project, new ProjectTable(table)); - } - - public boolean isExposedColumn(String project, String table, String col) { - project = ProjectInstance.getNormalizedProjectName(project); - - return getProjectTable(project, table).getColumns().contains(col); - } - - public List listAllCubes(String project) { - project = ProjectInstance.getNormalizedProjectName(project); - - HashSet ret = new HashSet(); - - ProjectInstance projectInstance = getProject(project); - if (projectInstance != null) { - for (String cubeName : projectInstance.getCubes()) { - CubeInstance cube = CubeManager.getInstance(config).getCube(cubeName); - if (null != cube) { - ret.add(cube); - } else { - logger.error("Failed to load cube " + cubeName); - } - } - } - - return new ArrayList(ret); - } - - - public List getCubesByTable(String project, String tableName) { - project = ProjectInstance.getNormalizedProjectName(project); - tableName = tableName.toUpperCase(); - List cubes = new ArrayList(); - - ProjectTable projectTable = getProjectTable(project, tableName); - cubes.addAll(projectTable.getCubes()); - - return cubes; - } - - public List getOnlineCubesByFactTable(String project, String factTableName) { - project = ProjectInstance.getNormalizedProjectName(project); - factTableName = factTableName.toUpperCase(); - List cubes = new ArrayList(); - ProjectTable projectTable = this.getProjectTable(project, factTableName); - for (CubeInstance cube : projectTable.getCubes()) { - if (cube.getDescriptor().getModel().isFactTable(factTableName) && cube.isReady()) { - cubes.add(cube); - } - } - - return cubes; - } - - public List listEffectiveRewriteMeasures(String project, String factTable) { - factTable = factTable.toUpperCase(); - - HashSet relatedDesc = new HashSet(); - for (CubeInstance cube : getProjectTable(project, factTable).getCubes()) { - if (cube.isReady() == false) - continue; - if (cube.getDescriptor().getModel().isFactTable(factTable) == false) - continue; - - relatedDesc.add(cube.getDescriptor()); - } - - List result = Lists.newArrayList(); - for (CubeDesc desc : relatedDesc) { - for (MeasureDesc m : desc.getMeasures()) { - FunctionDesc func = m.getFunction(); - if (func.needRewrite()) - result.add(m); - } - } - - return result; - } - - public void loadProjectCache(ProjectInstance project, boolean triggerUpdate) throws IOException { - loadProject(project.getResourcePath(), triggerUpdate); - loadTables(project.getResourcePath()); - } - - public void removeProjectCache(ProjectInstance project) { - String projectName = ProjectInstance.getNormalizedProjectName(project.getName()); - if (projectMap.containsKey(projectName)) { - projectMap.remove(projectName); - projectTables.removeAll(projectName); - } - } - - private void mapTableToCube(ProjectInstance projectInstance, CubeInstance cubeInstance) { - // schema sanity check - CubeDesc cubeDesc = cubeInstance.getDescriptor(); - if (cubeDesc == null) { - logger.warn("No CubeDesc found by name '" + cubeInstance.getDescName() + "'"); - return; - } - - // table ==> cube mapping - String factTable = cubeDesc.getFactTable(); - logger.debug("Fact Table: " + factTable + " -- Cube: " + cubeInstance.getName()); - assert this.getMetadataManager().getTableDesc(factTable) != null; - - String project = ProjectInstance.getNormalizedProjectName(projectInstance.getName()); - ProjectTable factProjTable = this.getProjectTable(project, factTable, true); - if (!factProjTable.getCubes().contains(cubeInstance)) { - factProjTable.getCubes().add(cubeInstance); - } - - for (DimensionDesc d : cubeDesc.getDimensions()) { - String lookupTable = d.getTable(); - logger.debug("Lookup Table: " + lookupTable + " -- Cube: " + cubeInstance.getName()); - assert this.getMetadataManager().getTableDesc(lookupTable) != null; - - ProjectTable dimensionProjTable = this.getProjectTable(project, lookupTable); - if (!dimensionProjTable.getCubes().contains(cubeInstance)) { - dimensionProjTable.getCubes().add(cubeInstance); - } - } - } - - private List findProjects(String cubeName) { - List projects = new ArrayList(); - for (ProjectInstance projectInstance : projectMap.values()) { - if (projectInstance.containsCube(cubeName)) { - projects.add(projectInstance); - } - } - - return projects; - } - - private synchronized ProjectInstance loadProject(String path, boolean triggerUpdate) throws IOException { - ResourceStore store = getStore(); - logger.debug("Loading CubeInstance " + store.getReadableResourcePath(path)); - - ProjectInstance projectInstance = store.getResource(path, ProjectInstance.class, PROJECT_SERIALIZER); - projectInstance.init(); - - if (StringUtils.isBlank(projectInstance.getName())) { - throw new IllegalStateException("Project name must not be blank"); - } - - if (triggerUpdate) { - projectMap.put(projectInstance.getName().toUpperCase(), projectInstance); - } else { - projectMap.putLocal(projectInstance.getName().toUpperCase(), projectInstance); - } - - return projectInstance; - } - - private synchronized void loadTables(String path) throws IOException { - ResourceStore store = getStore(); - logger.debug("Loading CubeInstance " + store.getReadableResourcePath(path)); - - ProjectInstance projectInstance = store.getResource(path, ProjectInstance.class, PROJECT_SERIALIZER); - projectInstance.init(); - - String project = ProjectInstance.getNormalizedProjectName(projectInstance.getName()); - projectTables.removeAll(project); - - for (CubeInstance cubeInstance : this.listAllCubes(projectInstance.getName())) { - markExposedTablesAndColumns(projectInstance.getName(), cubeInstance); - mapTableToCube(projectInstance, cubeInstance); - } - } - - private void loadAllProjects() throws IOException { - ResourceStore store = getStore(); - List paths = store.collectResourceRecursively(ResourceStore.PROJECT_RESOURCE_ROOT, ".json"); - - logger.debug("Loading Project from folder " + store.getReadableResourcePath(ResourceStore.PROJECT_RESOURCE_ROOT)); - - for (String path : paths) { - loadProject(path, false); - loadTables(path); - } - - logger.debug("Loaded " + paths.size() + " Project(s)"); - } - - private ProjectInstance addCubeToProject(String cubeName, String project, String user) throws IOException { - String newProjectName = ProjectInstance.getNormalizedProjectName(project); - ProjectInstance newProject = getProject(newProjectName); - if (newProject == null) { - newProject = this.createProject(newProjectName, user, "This is a project automatically added when adding cube " + cubeName); - } - newProject.addCube(cubeName); - saveResource(newProject); - - return newProject; - } - - private void saveResource(ProjectInstance proj) throws IOException { - ResourceStore store = getStore(); - store.putResource(proj.getResourcePath(), proj, PROJECT_SERIALIZER); - afterProjectUpdated(proj); - } - - private void deleteResource(ProjectInstance proj) throws IOException { - ResourceStore store = getStore(); - store.deleteResource(proj.getResourcePath()); - this.afterProjectDropped(proj); - } - - private void afterProjectUpdated(ProjectInstance updatedProject) { - try { - this.loadProjectCache(updatedProject, true); - } catch (IOException e) { - logger.error(e.getLocalizedMessage(), e); - } - } - - private void afterProjectDropped(ProjectInstance droppedProject) { - this.removeProjectCache(droppedProject); - } - - // sync on update - private void markExposedTablesAndColumns(String projectName, CubeInstance cubeInstance) { - if (!cubeInstance.isReady()) - return; - - CubeDesc cubeDesc = cubeInstance.getDescriptor(); - String factTable = cubeDesc.getFactTable(); - for (DimensionDesc dim : cubeDesc.getDimensions()) { - String lookupTable = dim.getTable(); - JoinDesc join = dim.getJoin(); - if (join == null) - continue; // for dimensions on fact table, there's no join - - if (join.getForeignKeyColumns() == null) { - throw new IllegalStateException("Null FK for " + join); - } - for (TblColRef fkCol : join.getForeignKeyColumns()) { - markExposedTableAndColumn(projectName, factTable, fkCol.getName(), dim); - } - if (join.getPrimaryKeyColumns() == null) { - throw new IllegalStateException("Null PK for " + join); - } - for (TblColRef pkCol : join.getPrimaryKeyColumns()) { - markExposedTableAndColumn(projectName, lookupTable, pkCol.getName(), dim); - } - } - for (TblColRef col : cubeDesc.listAllColumns()) { - markExposedTableAndColumn(projectName, col.getTable(), col.getName(), col); - } - } - - private void markExposedTableAndColumn(String project, String table, String column, Object refObj) { - project = ProjectInstance.getNormalizedProjectName(project); - TableDesc t = this.getMetadataManager().getTableDesc(table); - if (t == null) { - throw new IllegalStateException("No SourceTable found by name '" + table + "', ref by " + refObj); - } - table = t.getIdentity(); // ensures upper case - - ProjectTable projTable = getProjectTable(project, table, true); - - ColumnDesc srcCol = t.findColumnByName(column); - if (srcCol == null) { - throw new IllegalStateException("No SourceColumn found by name '" + table + "/" + column + "', ref by " + refObj); - } - - if (!projTable.getColumns().contains(srcCol.getName())) { - projTable.getColumns().add(srcCol.getName()); - } - } - - private ProjectTable getProjectTable(String project, final String table) { - return getProjectTable(project, table, false); - } - - private ProjectTable getProjectTable(String project, final String table, boolean autoCreate) { - ProjectTable projectTable = null; - project = ProjectInstance.getNormalizedProjectName(project); - - if (this.projectTables.containsEntry(project, new ProjectTable(table))) { - Collection projects = this.projectTables.get(project); - for (ProjectTable pt : projects) { - if (pt.getName().equalsIgnoreCase(table)) { - projectTable = pt; - break; - } - } - } else { - projectTable = new ProjectTable(table); - - if (autoCreate) { - this.projectTables.put(project, projectTable); - } - } - - return projectTable; - } - - private ResourceStore getStore() { - return ResourceStore.getStore(this.config); - } - - private MetadataManager getMetadataManager() { - return MetadataManager.getInstance(config); - } -} diff --git a/job/src/main/java/com/kylinolap/job/JobManager.java b/job/src/main/java/com/kylinolap/job/JobManager.java index 8b94693..eabd9f8 100644 --- a/job/src/main/java/com/kylinolap/job/JobManager.java +++ b/job/src/main/java/com/kylinolap/job/JobManager.java @@ -20,7 +20,6 @@ import com.kylinolap.cube.*; import com.kylinolap.cube.exception.CubeIntegrityException; import com.kylinolap.cube.model.CubeDesc; -import com.kylinolap.cube.project.ProjectManager; import com.kylinolap.job.JobInstance.JobStep; import com.kylinolap.job.constant.JobConstants; import com.kylinolap.job.constant.JobStatusEnum; @@ -31,6 +30,7 @@ import com.kylinolap.job.exception.JobException; import com.kylinolap.job.hadoop.hive.JoinedFlatTableDesc; import com.kylinolap.metadata.project.ProjectInstance; +import com.kylinolap.metadata.project.ProjectManager; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/metadata/src/main/java/com/kylinolap/metadata/project/ProjectManager.java b/metadata/src/main/java/com/kylinolap/metadata/project/ProjectManager.java index db4b394..42bbefa 100644 --- a/metadata/src/main/java/com/kylinolap/metadata/project/ProjectManager.java +++ b/metadata/src/main/java/com/kylinolap/metadata/project/ProjectManager.java @@ -169,6 +169,18 @@ public ProjectInstance updateProject(ProjectInstance project, String newName, St } } + public void removeCubeFromProjects(String cubeName) throws IOException { + for (ProjectInstance projectInstance : findProjects(cubeName)) { + projectInstance.removeCube(cubeName); + saveResource(projectInstance); + } + } + + public ProjectInstance updateCubeToProject(String cubeName, String newProjectName, String owner) throws IOException { + removeCubeFromProjects(cubeName); + return addCubeToProject(cubeName, newProjectName, owner); + } + public boolean containsRealization(String projectName, DataModelRealizationType realization, String name) { ProjectInstance project = getProject(projectName); if (project == null) { diff --git a/query/src/main/java/com/kylinolap/query/routing/QueryRouter.java b/query/src/main/java/com/kylinolap/query/routing/QueryRouter.java index 4e5bbde..d513d67 100644 --- a/query/src/main/java/com/kylinolap/query/routing/QueryRouter.java +++ b/query/src/main/java/com/kylinolap/query/routing/QueryRouter.java @@ -24,6 +24,7 @@ import java.util.List; import java.util.Map; +import com.kylinolap.cube.project.CubeRealizationManager; import org.apache.commons.lang3.StringUtils; import org.eigenbase.reltype.RelDataType; import org.slf4j.Logger; @@ -33,7 +34,6 @@ import com.kylinolap.cube.CubeManager; import com.kylinolap.cube.model.CubeDesc; import com.kylinolap.cube.model.DimensionDesc; -import com.kylinolap.cube.project.ProjectManager; import com.kylinolap.metadata.model.JoinDesc; import com.kylinolap.metadata.model.realization.FunctionDesc; import com.kylinolap.metadata.model.realization.ParameterDesc; @@ -51,18 +51,18 @@ public static CubeInstance findCube(OLAPContext olapContext) throws CubeNotFound CubeInstance bestCube = null; // NOTE: since some query has no groups and projections are the superset of groups, we choose projections. - ProjectManager projectManager = ProjectManager.getInstance(olapContext.olapSchema.getConfig()); + CubeRealizationManager cubeRealizationManager = CubeRealizationManager.getInstance(olapContext.olapSchema.getConfig()); if (olapContext.isSimpleQuery()) { // if simple query like "select X from fact table", just return the cube with most dimensions // Note that this will only succeed to get best cube if the current simple query is on fact table. // Simple query on look up table is handled in OLAPTableScan.genExecFunc // In other words, for simple query on lookup tables, bestCube here will be assigned null in this method - bestCube = findCubeWithMostDimensions(projectManager, olapContext); + bestCube = findCubeWithMostDimensions(cubeRealizationManager, olapContext); } if (bestCube == null) { - bestCube = findBestMatchCube(projectManager, olapContext); + bestCube = findBestMatchCube(cubeRealizationManager, olapContext); } if (bestCube == null) { @@ -75,7 +75,7 @@ public static CubeInstance findCube(OLAPContext olapContext) throws CubeNotFound return bestCube; } - private static CubeInstance findCubeWithMostDimensions(ProjectManager projectManager, OLAPContext olapContext) { + private static CubeInstance findCubeWithMostDimensions(CubeRealizationManager projectManager, OLAPContext olapContext) { List candidates = projectManager.getOnlineCubesByFactTable(olapContext.olapSchema.getProjectName(), olapContext.firstTableScan.getCubeTable()); if (candidates.isEmpty()) { return null; @@ -130,7 +130,7 @@ public int compare(CubeInstance c1, CubeInstance c2) { return null; } - static CubeInstance findBestMatchCube(ProjectManager projectManager, OLAPContext olapContext) throws CubeNotFoundException { + static CubeInstance findBestMatchCube(CubeRealizationManager cubeRealizationManager, OLAPContext olapContext) throws CubeNotFoundException { // retrieve members from olapContext String factTableName = olapContext.firstTableScan.getCubeTable(); @@ -142,7 +142,7 @@ static CubeInstance findBestMatchCube(ProjectManager projectManager, OLAPContext Map rewriteFields = olapContext.rewriteFields; // find cubes by table - List candidates = projectManager.getCubesByTable(projectName, factTableName); + List candidates = cubeRealizationManager.getCubesByTable(projectName, factTableName); logger.info("Find candidates by table " + factTableName + " and project=" + projectName + " : " + StringUtils.join(candidates, ",")); // match dimensions & aggregations & joins diff --git a/query/src/main/java/com/kylinolap/query/schema/OLAPSchema.java b/query/src/main/java/com/kylinolap/query/schema/OLAPSchema.java index da45eb7..7fccfac 100644 --- a/query/src/main/java/com/kylinolap/query/schema/OLAPSchema.java +++ b/query/src/main/java/com/kylinolap/query/schema/OLAPSchema.java @@ -19,13 +19,13 @@ import java.util.List; import java.util.Map; +import com.kylinolap.cube.project.CubeRealizationManager; import com.kylinolap.metadata.project.ProjectInstance; import net.hydromatic.optiq.Table; import net.hydromatic.optiq.impl.AbstractSchema; import com.kylinolap.common.KylinConfig; import com.kylinolap.cube.CubeManager; -import com.kylinolap.cube.project.ProjectManager; import com.kylinolap.metadata.MetadataManager; import com.kylinolap.metadata.model.TableDesc; @@ -70,7 +70,7 @@ public OLAPSchema(String project, String schemaName) { private Map buildTableMap() { Map olapTables = new HashMap(); - List projectTables = getProjectManager().listExposedTables(projectName); + List projectTables = CubeRealizationManager.getInstance(config).listExposedTables(projectName); for (TableDesc tableDesc : projectTables) { if (tableDesc.getDatabase().equals(schemaName)) { @@ -124,7 +124,4 @@ public CubeManager getCubeManager() { return CubeManager.getInstance(config); } - public ProjectManager getProjectManager() { - return ProjectManager.getInstance(config); - } } diff --git a/query/src/main/java/com/kylinolap/query/schema/OLAPSchemaFactory.java b/query/src/main/java/com/kylinolap/query/schema/OLAPSchemaFactory.java index dd5fc7f..879f398 100644 --- a/query/src/main/java/com/kylinolap/query/schema/OLAPSchemaFactory.java +++ b/query/src/main/java/com/kylinolap/query/schema/OLAPSchemaFactory.java @@ -22,6 +22,7 @@ import java.util.List; import java.util.Map; +import com.kylinolap.cube.project.CubeRealizationManager; import com.kylinolap.metadata.project.ProjectInstance; import net.hydromatic.optiq.Schema; import net.hydromatic.optiq.SchemaFactory; @@ -30,7 +31,6 @@ import org.eigenbase.util14.ConversionUtil; import com.kylinolap.common.KylinConfig; -import com.kylinolap.cube.project.ProjectManager; import com.kylinolap.metadata.model.DatabaseDesc; import com.kylinolap.metadata.model.TableDesc; @@ -68,7 +68,7 @@ public Schema create(SchemaPlus parentSchema, String schemaName, Map tables = ProjectManager.getInstance(config).listExposedTables(project); + List tables = CubeRealizationManager.getInstance(config).listExposedTables(project); // "database" in TableDesc correspond to our schema // the logic to decide which schema to be "default" in calcite: diff --git a/query/src/main/java/com/kylinolap/query/schema/OLAPTable.java b/query/src/main/java/com/kylinolap/query/schema/OLAPTable.java index d01bb9a..bc90793 100644 --- a/query/src/main/java/com/kylinolap/query/schema/OLAPTable.java +++ b/query/src/main/java/com/kylinolap/query/schema/OLAPTable.java @@ -22,6 +22,7 @@ import java.util.List; import java.util.Map; +import com.kylinolap.cube.project.CubeRealizationManager; import net.hydromatic.linq4j.Enumerable; import net.hydromatic.linq4j.Enumerator; import net.hydromatic.linq4j.QueryProvider; @@ -43,7 +44,6 @@ import org.eigenbase.sql.type.SqlTypeUtil; import com.kylinolap.cube.model.MeasureDesc; -import com.kylinolap.cube.project.ProjectManager; import com.kylinolap.metadata.model.ColumnDesc; import com.kylinolap.metadata.model.TableDesc; import com.kylinolap.metadata.model.realization.FunctionDesc; @@ -161,7 +161,7 @@ else if (precision >= 0) } private List listSourceColumns() { - ProjectManager projectMgr = olapSchema.getProjectManager(); + CubeRealizationManager projectMgr = CubeRealizationManager.getInstance(olapSchema.getConfig()); List exposedColumns = projectMgr.listExposedColumns(olapSchema.getProjectName(), sourceTable.getIdentity()); List countMeasures = projectMgr.listEffectiveRewriteMeasures(olapSchema.getProjectName(), sourceTable.getIdentity()); diff --git a/server/src/main/java/com/kylinolap/rest/controller/TableController.java b/server/src/main/java/com/kylinolap/rest/controller/TableController.java index 69e0848..9b036aa 100644 --- a/server/src/main/java/com/kylinolap/rest/controller/TableController.java +++ b/server/src/main/java/com/kylinolap/rest/controller/TableController.java @@ -70,7 +70,7 @@ long start = System.currentTimeMillis(); List tables = null; try { - tables = cubeMgmtService.getProjectManager().listDefinedTablesInProject(project); + tables = cubeMgmtService.getProjectManager().listDefinedTablesInProject(project); } catch (Exception e) { logger.error("Failed to deal with the request.", e); throw new InternalErrorException(e.getLocalizedMessage()); diff --git a/server/src/main/java/com/kylinolap/rest/service/BasicService.java b/server/src/main/java/com/kylinolap/rest/service/BasicService.java index 87f5eae..92b6cb7 100644 --- a/server/src/main/java/com/kylinolap/rest/service/BasicService.java +++ b/server/src/main/java/com/kylinolap/rest/service/BasicService.java @@ -31,6 +31,7 @@ import javax.sql.DataSource; +import com.kylinolap.cube.project.CubeRealizationManager; import com.kylinolap.metadata.project.ProjectInstance; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; @@ -43,7 +44,6 @@ import com.kylinolap.common.KylinConfig; import com.kylinolap.cube.CubeDescManager; import com.kylinolap.cube.CubeManager; -import com.kylinolap.cube.project.ProjectManager; import com.kylinolap.job.JobManager; import com.kylinolap.job.engine.JobEngineConfig; import com.kylinolap.job.exception.JobException; @@ -118,20 +118,18 @@ public DataSource getOLAPDataSource(String project) { /** * Reload changed cube into cache * - * @param name * @throws IOException */ @Caching(evict = { @CacheEvict(value = QueryController.SUCCESS_QUERY_CACHE, allEntries = true), @CacheEvict(value = QueryController.EXCEPTION_QUERY_CACHE, allEntries = true) }) public void cleanDataCache() { CubeManager.removeInstance(getConfig()); - ProjectManager.removeInstance(getConfig()); + CubeRealizationManager.removeInstance(getConfig()); BasicService.resetOLAPDataSources(); } /** * Reload the cube desc with name {name} into cache * - * @param name */ public void reloadMetadataCache() { MetadataManager.getInstance(getConfig()).reload(); @@ -159,8 +157,8 @@ public CubeDescManager getCubeDescManager() { return CubeDescManager.getInstance(getConfig()); } - public ProjectManager getProjectManager() { - return ProjectManager.getInstance(getConfig()); + public CubeRealizationManager getProjectManager() { + return CubeRealizationManager.getInstance(getConfig()); } public JobManager getJobManager() throws JobException, UnknownHostException { From 54f8a30b24c30e86b0f4c711682e9b7a00f3af09 Mon Sep 17 00:00:00 2001 From: "qianhao.zhou" Date: Mon, 8 Dec 2014 17:44:48 +0800 Subject: [PATCH 4/4] fix compile issue --- .../cube/project/CubeRealizationManager.java | 23 ++- .../com/kylinolap/cube/CubeManagerCacheTest.java | 4 +- .../java/com/kylinolap/cube/CubeManagerTest.java | 8 +- .../com/kylinolap/cube/SegmentManagementTest.java | 4 +- .../cube/project/CubeRealizationManagerTest.java | 180 +++++++++++++++++++++ .../kylinolap/cube/project/ProjectManagerTest.java | 176 -------------------- .../kylinolap/job/engine/GenericJobEngineTest.java | 4 +- .../job/engine/JobInstanceBuilderTest.java | 4 +- .../job/hadoop/cube/MergeCuboidMapperTest.java | 4 +- .../kylinolap/rest/controller/TableController.java | 2 +- .../com/kylinolap/rest/service/BasicService.java | 7 +- .../com/kylinolap/rest/service/CubeService.java | 50 +++++- .../com/kylinolap/rest/service/QueryService.java | 10 +- .../rest/controller/ProjectControllerTest.java | 2 +- .../kylinolap/rest/service/ServiceTestBase.java | 4 +- 15 files changed, 263 insertions(+), 219 deletions(-) create mode 100644 cube/src/test/java/com/kylinolap/cube/project/CubeRealizationManagerTest.java delete mode 100644 cube/src/test/java/com/kylinolap/cube/project/ProjectManagerTest.java diff --git a/cube/src/main/java/com/kylinolap/cube/project/CubeRealizationManager.java b/cube/src/main/java/com/kylinolap/cube/project/CubeRealizationManager.java index 956000f..a43ca67 100644 --- a/cube/src/main/java/com/kylinolap/cube/project/CubeRealizationManager.java +++ b/cube/src/main/java/com/kylinolap/cube/project/CubeRealizationManager.java @@ -181,11 +181,10 @@ public boolean isCubeInProject(String projectName, CubeInstance cube) { return this.listAllCubes(projectName).contains(cube); } - public ProjectInstance updateCubeToProject(String cubeName, String newProjectName, String owner) throws IOException { - removeCubeFromProjects(cubeName); - - return addCubeToProject(cubeName, newProjectName, owner); - } +// public ProjectInstance updateCubeToProject(String cubeName, String newProjectName, String owner) throws IOException { +// removeCubeFromProjects(cubeName); +// return addCubeToProject(cubeName, newProjectName, owner); +// } public ProjectInstance updateTableToProject(String tables, String projectName) throws IOException { ProjectInstance projectInstance = getProject(projectName); @@ -207,13 +206,13 @@ public ProjectInstance updateTableToProject(String tables, String projectName) t } - public void removeCubeFromProjects(String cubeName) throws IOException { - for (ProjectInstance projectInstance : findProjects(cubeName)) { - projectInstance.removeCube(cubeName); - - saveResource(projectInstance); - } - } +// public void removeCubeFromProjects(String cubeName) throws IOException { +// for (ProjectInstance projectInstance : findProjects(cubeName)) { +// projectInstance.removeCube(cubeName); +// +// saveResource(projectInstance); +// } +// } public List listExposedTables(String project) { project = ProjectInstance.getNormalizedProjectName(project); diff --git a/cube/src/test/java/com/kylinolap/cube/CubeManagerCacheTest.java b/cube/src/test/java/com/kylinolap/cube/CubeManagerCacheTest.java index c9f9b24..cfed258 100644 --- a/cube/src/test/java/com/kylinolap/cube/CubeManagerCacheTest.java +++ b/cube/src/test/java/com/kylinolap/cube/CubeManagerCacheTest.java @@ -18,6 +18,7 @@ import static org.junit.Assert.*; +import com.kylinolap.cube.project.CubeRealizationManager; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -25,7 +26,6 @@ import com.kylinolap.common.persistence.ResourceStore; import com.kylinolap.common.util.LocalFileMetadataTestCase; import com.kylinolap.cube.model.CubeDesc; -import com.kylinolap.cube.project.ProjectManager; import com.kylinolap.metadata.MetadataManager; /** @@ -41,7 +41,7 @@ public void setUp() throws Exception { this.createTestMetadata(); MetadataManager.removeInstance(this.getTestConfig()); CubeManager.removeInstance(this.getTestConfig()); - ProjectManager.removeInstance(this.getTestConfig()); + CubeRealizationManager.removeInstance(this.getTestConfig()); cubeManager = CubeManager.getInstance(this.getTestConfig()); } diff --git a/cube/src/test/java/com/kylinolap/cube/CubeManagerTest.java b/cube/src/test/java/com/kylinolap/cube/CubeManagerTest.java index b9e778c..c0e17d0 100644 --- a/cube/src/test/java/com/kylinolap/cube/CubeManagerTest.java +++ b/cube/src/test/java/com/kylinolap/cube/CubeManagerTest.java @@ -18,6 +18,7 @@ import static org.junit.Assert.*; +import com.kylinolap.cube.project.CubeRealizationManager; import com.kylinolap.metadata.project.ProjectInstance; import org.junit.After; import org.junit.Before; @@ -27,7 +28,6 @@ import com.kylinolap.common.util.JsonUtil; import com.kylinolap.common.util.LocalFileMetadataTestCase; import com.kylinolap.cube.model.CubeDesc; -import com.kylinolap.cube.project.ProjectManager; import com.kylinolap.metadata.MetadataManager; /** @@ -40,7 +40,7 @@ public void setUp() throws Exception { this.createTestMetadata(); MetadataManager.removeInstance(this.getTestConfig()); CubeManager.removeInstance(this.getTestConfig()); - ProjectManager.removeInstance(this.getTestConfig()); + CubeRealizationManager.removeInstance(this.getTestConfig()); } @After @@ -72,12 +72,12 @@ public void testCreateAndDrop() throws Exception { CubeInstance createdCube = CubeManager.getInstance(this.getTestConfig()).createCube("a_whole_new_cube", ProjectInstance.DEFAULT_PROJECT_NAME, desc, null); assertTrue(createdCube == CubeManager.getInstance(this.getTestConfig()).getCube("a_whole_new_cube")); - assertTrue(ProjectManager.getInstance(getTestConfig()).listAllCubes(ProjectInstance.DEFAULT_PROJECT_NAME).contains(createdCube)); + assertTrue(CubeRealizationManager.getInstance(getTestConfig()).listAllCubes(ProjectInstance.DEFAULT_PROJECT_NAME).contains(createdCube)); CubeInstance droppedCube = CubeManager.getInstance(this.getTestConfig()).dropCube("a_whole_new_cube", true); assertTrue(createdCube == droppedCube); - assertTrue(!ProjectManager.getInstance(getTestConfig()).listAllCubes(ProjectInstance.DEFAULT_PROJECT_NAME).contains(droppedCube)); + assertTrue(!CubeRealizationManager.getInstance(getTestConfig()).listAllCubes(ProjectInstance.DEFAULT_PROJECT_NAME).contains(droppedCube)); assertNull(CubeManager.getInstance(this.getTestConfig()).getCube("a_whole_new_cube")); } diff --git a/cube/src/test/java/com/kylinolap/cube/SegmentManagementTest.java b/cube/src/test/java/com/kylinolap/cube/SegmentManagementTest.java index 298e45f..93c72b7 100644 --- a/cube/src/test/java/com/kylinolap/cube/SegmentManagementTest.java +++ b/cube/src/test/java/com/kylinolap/cube/SegmentManagementTest.java @@ -34,7 +34,7 @@ import com.kylinolap.common.util.LocalFileMetadataTestCase; import com.kylinolap.cube.exception.CubeIntegrityException; import com.kylinolap.cube.model.CubeDesc; -import com.kylinolap.cube.project.ProjectManager; +import com.kylinolap.cube.project.CubeRealizationManager; import com.kylinolap.metadata.MetadataManager; /** @@ -50,7 +50,7 @@ public void setUp() throws Exception { this.createTestMetadata(); MetadataManager.removeInstance(this.getTestConfig()); CubeManager.removeInstance(this.getTestConfig()); - ProjectManager.removeInstance(this.getTestConfig()); + CubeRealizationManager.removeInstance(this.getTestConfig()); cubeMgr = CubeManager.getInstance(this.getTestConfig()); } diff --git a/cube/src/test/java/com/kylinolap/cube/project/CubeRealizationManagerTest.java b/cube/src/test/java/com/kylinolap/cube/project/CubeRealizationManagerTest.java new file mode 100644 index 0000000..90b2efd --- /dev/null +++ b/cube/src/test/java/com/kylinolap/cube/project/CubeRealizationManagerTest.java @@ -0,0 +1,180 @@ +/* + * Copyright 2013-2014 eBay Software Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kylinolap.cube.project; + +import static org.junit.Assert.*; + +import java.io.IOException; + +import com.kylinolap.metadata.project.ProjectInstance; +import com.kylinolap.metadata.project.ProjectManager; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import com.kylinolap.common.persistence.ResourceStore; +import com.kylinolap.common.util.JsonUtil; +import com.kylinolap.common.util.LocalFileMetadataTestCase; +import com.kylinolap.cube.CubeDescManager; +import com.kylinolap.cube.CubeInstance; +import com.kylinolap.cube.CubeManager; +import com.kylinolap.cube.model.CubeDesc; +import com.kylinolap.metadata.MetadataManager; + +/** + * @author xduo + * + */ +public class CubeRealizationManagerTest extends LocalFileMetadataTestCase { + + @Before + public void setUp() throws Exception { + this.createTestMetadata(); + MetadataManager.removeInstance(this.getTestConfig()); + CubeManager.removeInstance(this.getTestConfig()); + CubeRealizationManager.removeInstance(this.getTestConfig()); + } + + @After + public void after() throws Exception { + this.cleanupTestMetadata(); + } + + @Test(expected = IllegalStateException.class) + public void testDropNonemptyProject1() throws IOException { + ProjectManager.getInstance(this.getTestConfig()).dropProject("DEFAULT"); + } + + @Test(expected = IllegalStateException.class) + public void testDropNonemptyProject2() throws IOException { + ProjectManager.getInstance(this.getTestConfig()).dropProject("DEFAULT???"); + } + + @Test + public void testNewProject() throws Exception { + ProjectManager projectmanager = ProjectManager.getInstance(this.getTestConfig()); + CubeRealizationManager cubeRealizationManager = CubeRealizationManager.getInstance(this.getTestConfig()); + int originalProjectCount = projectmanager.listAllProjects().size(); + int originalCubeCount = CubeManager.getInstance(this.getTestConfig()).listAllCubes().size(); + int originalCubeCountInDefault = cubeRealizationManager.listAllCubes("default").size(); + ResourceStore store = getStore(); + + // clean legacy in case last run failed + store.deleteResource("/cube/cube_in_alien_project.json"); + + CubeDescManager cubeDescMgr = getCubeDescManager(); + CubeDesc desc = cubeDescMgr.getCubeDesc("test_kylin_cube_with_slr_desc"); + CubeInstance createdCube = CubeManager.getInstance(this.getTestConfig()).createCube("cube_in_alien_project", "alien", desc, null); + assertTrue(createdCube == CubeManager.getInstance(this.getTestConfig()).getCube("cube_in_alien_project")); + assertTrue(CubeRealizationManager.getInstance(getTestConfig()).listAllCubes("alien").contains(createdCube)); + + System.out.println(JsonUtil.writeValueAsIndentString(createdCube)); + + assertTrue(projectmanager.listAllProjects().size() == originalProjectCount + 1); + assertTrue(cubeRealizationManager.listAllCubes("ALIEN").get(0).getName().equalsIgnoreCase("CUBE_IN_ALIEN_PROJECT")); + assertTrue(CubeManager.getInstance(this.getTestConfig()).listAllCubes().size() == originalCubeCount + 1); + + projectmanager.updateCubeToProject("cube_in_alien_project", "default", null); + assertTrue(cubeRealizationManager.listAllCubes("ALIEN").size() == 0); + assertTrue(cubeRealizationManager.listAllCubes("default").size() == originalCubeCountInDefault + 1); + assertTrue(cubeRealizationManager.listAllCubes("default").contains(createdCube)); + + projectmanager.updateCubeToProject("cube_in_alien_project", "alien", null); + assertTrue(cubeRealizationManager.listAllCubes("ALIEN").size() == 1); + assertTrue(cubeRealizationManager.listAllCubes("default").size() == originalCubeCountInDefault); + assertTrue(cubeRealizationManager.listAllCubes("alien").contains(createdCube)); + + assertTrue(cubeRealizationManager.isCubeInProject("alien", createdCube)); + + CubeInstance droppedCube = CubeManager.getInstance(this.getTestConfig()).dropCube("cube_in_alien_project", true); + + assertTrue(createdCube == droppedCube); + assertNull(CubeManager.getInstance(this.getTestConfig()).getCube("cube_in_alien_project")); + assertTrue(projectmanager.listAllProjects().size() == originalProjectCount + 1); + assertTrue(CubeManager.getInstance(this.getTestConfig()).listAllCubes().size() == originalCubeCount); + + projectmanager.dropProject("alien"); + assertTrue(projectmanager.listAllProjects().size() == originalProjectCount); + } + + @Test + public void testExistingProject() throws Exception { + int originalProjectCount = ProjectManager.getInstance(this.getTestConfig()).listAllProjects().size(); + int originalCubeCount = CubeManager.getInstance(this.getTestConfig()).listAllCubes().size(); + ResourceStore store = getStore(); + + // clean legacy in case last run failed + store.deleteResource("/cube/new_cube_in_default.json"); + + CubeDescManager cubeDescMgr = getCubeDescManager(); + CubeDesc desc = cubeDescMgr.getCubeDesc("test_kylin_cube_with_slr_desc"); + CubeInstance createdCube = CubeManager.getInstance(this.getTestConfig()).createCube("new_cube_in_default", ProjectInstance.DEFAULT_PROJECT_NAME, desc, null); + assertTrue(createdCube == CubeManager.getInstance(this.getTestConfig()).getCube("new_cube_in_default")); + + System.out.println(JsonUtil.writeValueAsIndentString(createdCube)); + + assertTrue(ProjectManager.getInstance(this.getTestConfig()).listAllProjects().size() == originalProjectCount); + assertTrue(CubeManager.getInstance(this.getTestConfig()).listAllCubes().size() == originalCubeCount + 1); + + CubeInstance droppedCube = CubeManager.getInstance(this.getTestConfig()).dropCube("new_cube_in_default", true); + + assertTrue(createdCube == droppedCube); + assertNull(CubeManager.getInstance(this.getTestConfig()).getCube("new_cube_in_default")); + assertTrue(ProjectManager.getInstance(this.getTestConfig()).listAllProjects().size() == originalProjectCount); + assertTrue(CubeManager.getInstance(this.getTestConfig()).listAllCubes().size() == originalCubeCount); + } + + @Test + public void testProjectsDrop() throws IOException { + CubeRealizationManager cubeRealizationManager = CubeRealizationManager.getInstance(this.getTestConfig()); + CubeInstance cube = CubeManager.getInstance(getTestConfig()).getCube("test_kylin_cube_with_slr_empty"); + assertTrue(cubeRealizationManager.getCubesByTable("default", "default.test_kylin_fact").contains(cube)); + assertTrue(cubeRealizationManager.listAllCubes("default").contains(cube)); + + CubeManager.getInstance(getTestConfig()).dropCube(cube.getName(), true); + + assertTrue(!cubeRealizationManager.getCubesByTable("default", "default.test_kylin_fact").contains(cube)); + assertTrue(!cubeRealizationManager.listAllCubes("default").contains(cube)); + } + + @Test + public void testProjectsLoadAfterProjectChange() throws IOException { + CubeRealizationManager cubeRealizationManager = CubeRealizationManager.getInstance(this.getTestConfig()); + CubeInstance cube = CubeManager.getInstance(getTestConfig()).getCube("test_kylin_cube_with_slr_empty"); + assertTrue(cubeRealizationManager.getCubesByTable("default", "default.test_kylin_fact").contains(cube)); + + ProjectManager.getInstance(getTestConfig()).removeCubeFromProjects(cube.getName()); + + assertTrue(!cubeRealizationManager.getCubesByTable("default", "default.test_kylin_fact").contains(cube)); + + ProjectManager.getInstance(getTestConfig()).updateCubeToProject(cube.getName(), "default", "tester"); + + assertTrue(cubeRealizationManager.getCubesByTable("default", "default.test_kylin_fact").contains(cube)); + } + + private MetadataManager getMetadataManager() { + return MetadataManager.getInstance(getTestConfig()); + } + + private CubeManager getCubeManager() { + return CubeManager.getInstance(getTestConfig()); + } + + public CubeDescManager getCubeDescManager() { + return CubeDescManager.getInstance(getTestConfig()); + } +} diff --git a/cube/src/test/java/com/kylinolap/cube/project/ProjectManagerTest.java b/cube/src/test/java/com/kylinolap/cube/project/ProjectManagerTest.java deleted file mode 100644 index b7fadfa..0000000 --- a/cube/src/test/java/com/kylinolap/cube/project/ProjectManagerTest.java +++ /dev/null @@ -1,176 +0,0 @@ -/* - * Copyright 2013-2014 eBay Software Foundation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.kylinolap.cube.project; - -import static org.junit.Assert.*; - -import java.io.IOException; - -import com.kylinolap.metadata.project.ProjectInstance; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -import com.kylinolap.common.persistence.ResourceStore; -import com.kylinolap.common.util.JsonUtil; -import com.kylinolap.common.util.LocalFileMetadataTestCase; -import com.kylinolap.cube.CubeDescManager; -import com.kylinolap.cube.CubeInstance; -import com.kylinolap.cube.CubeManager; -import com.kylinolap.cube.model.CubeDesc; -import com.kylinolap.metadata.MetadataManager; - -/** - * @author xduo - * - */ -public class ProjectManagerTest extends LocalFileMetadataTestCase { - - @Before - public void setUp() throws Exception { - this.createTestMetadata(); - MetadataManager.removeInstance(this.getTestConfig()); - CubeManager.removeInstance(this.getTestConfig()); - ProjectManager.removeInstance(this.getTestConfig()); - } - - @After - public void after() throws Exception { - this.cleanupTestMetadata(); - } - - @Test(expected = IllegalStateException.class) - public void testDropNonemptyProject1() throws IOException { - ProjectManager.getInstance(this.getTestConfig()).dropProject("DEFAULT"); - } - - @Test(expected = IllegalStateException.class) - public void testDropNonemptyProject2() throws IOException { - ProjectManager.getInstance(this.getTestConfig()).dropProject("DEFAULT???"); - } - - @Test - public void testNewProject() throws Exception { - ProjectManager projectmanager = ProjectManager.getInstance(this.getTestConfig()); - int originalProjectCount = projectmanager.listAllProjects().size(); - int originalCubeCount = CubeManager.getInstance(this.getTestConfig()).listAllCubes().size(); - int originalCubeCountInDefault = projectmanager.listAllCubes("default").size(); - ResourceStore store = getStore(); - - // clean legacy in case last run failed - store.deleteResource("/cube/cube_in_alien_project.json"); - - CubeDescManager cubeDescMgr = getCubeDescManager(); - CubeDesc desc = cubeDescMgr.getCubeDesc("test_kylin_cube_with_slr_desc"); - CubeInstance createdCube = CubeManager.getInstance(this.getTestConfig()).createCube("cube_in_alien_project", "alien", desc, null); - assertTrue(createdCube == CubeManager.getInstance(this.getTestConfig()).getCube("cube_in_alien_project")); - assertTrue(ProjectManager.getInstance(getTestConfig()).listAllCubes("alien").contains(createdCube)); - - System.out.println(JsonUtil.writeValueAsIndentString(createdCube)); - - assertTrue(projectmanager.listAllProjects().size() == originalProjectCount + 1); - assertTrue(projectmanager.listAllCubes("ALIEN").get(0).getName().equalsIgnoreCase("CUBE_IN_ALIEN_PROJECT")); - assertTrue(CubeManager.getInstance(this.getTestConfig()).listAllCubes().size() == originalCubeCount + 1); - - projectmanager.updateCubeToProject("cube_in_alien_project", "default", null); - assertTrue(projectmanager.listAllCubes("ALIEN").size() == 0); - assertTrue(projectmanager.listAllCubes("default").size() == originalCubeCountInDefault + 1); - assertTrue(ProjectManager.getInstance(getTestConfig()).listAllCubes("default").contains(createdCube)); - - projectmanager.updateCubeToProject("cube_in_alien_project", "alien", null); - assertTrue(projectmanager.listAllCubes("ALIEN").size() == 1); - assertTrue(projectmanager.listAllCubes("default").size() == originalCubeCountInDefault); - assertTrue(ProjectManager.getInstance(getTestConfig()).listAllCubes("alien").contains(createdCube)); - - assertTrue(projectmanager.isCubeInProject("alien", createdCube)); - - CubeInstance droppedCube = CubeManager.getInstance(this.getTestConfig()).dropCube("cube_in_alien_project", true); - - assertTrue(createdCube == droppedCube); - assertNull(CubeManager.getInstance(this.getTestConfig()).getCube("cube_in_alien_project")); - assertTrue(projectmanager.listAllProjects().size() == originalProjectCount + 1); - assertTrue(CubeManager.getInstance(this.getTestConfig()).listAllCubes().size() == originalCubeCount); - - projectmanager.dropProject("alien"); - assertTrue(projectmanager.listAllProjects().size() == originalProjectCount); - } - - @Test - public void testExistingProject() throws Exception { - int originalProjectCount = ProjectManager.getInstance(this.getTestConfig()).listAllProjects().size(); - int originalCubeCount = CubeManager.getInstance(this.getTestConfig()).listAllCubes().size(); - ResourceStore store = getStore(); - - // clean legacy in case last run failed - store.deleteResource("/cube/new_cube_in_default.json"); - - CubeDescManager cubeDescMgr = getCubeDescManager(); - CubeDesc desc = cubeDescMgr.getCubeDesc("test_kylin_cube_with_slr_desc"); - CubeInstance createdCube = CubeManager.getInstance(this.getTestConfig()).createCube("new_cube_in_default", ProjectInstance.DEFAULT_PROJECT_NAME, desc, null); - assertTrue(createdCube == CubeManager.getInstance(this.getTestConfig()).getCube("new_cube_in_default")); - - System.out.println(JsonUtil.writeValueAsIndentString(createdCube)); - - assertTrue(ProjectManager.getInstance(this.getTestConfig()).listAllProjects().size() == originalProjectCount); - assertTrue(CubeManager.getInstance(this.getTestConfig()).listAllCubes().size() == originalCubeCount + 1); - - CubeInstance droppedCube = CubeManager.getInstance(this.getTestConfig()).dropCube("new_cube_in_default", true); - - assertTrue(createdCube == droppedCube); - assertNull(CubeManager.getInstance(this.getTestConfig()).getCube("new_cube_in_default")); - assertTrue(ProjectManager.getInstance(this.getTestConfig()).listAllProjects().size() == originalProjectCount); - assertTrue(CubeManager.getInstance(this.getTestConfig()).listAllCubes().size() == originalCubeCount); - } - - @Test - public void testProjectsDrop() throws IOException { - CubeInstance cube = CubeManager.getInstance(getTestConfig()).getCube("test_kylin_cube_with_slr_empty"); - assertTrue(ProjectManager.getInstance(this.getTestConfig()).getCubesByTable("default", "default.test_kylin_fact").contains(cube)); - assertTrue(ProjectManager.getInstance(this.getTestConfig()).listAllCubes("default").contains(cube)); - - CubeManager.getInstance(getTestConfig()).dropCube(cube.getName(), true); - - assertTrue(!ProjectManager.getInstance(this.getTestConfig()).getCubesByTable("default", "default.test_kylin_fact").contains(cube)); - assertTrue(!ProjectManager.getInstance(this.getTestConfig()).listAllCubes("default").contains(cube)); - } - - @Test - public void testProjectsLoadAfterProjectChange() throws IOException { - CubeInstance cube = CubeManager.getInstance(getTestConfig()).getCube("test_kylin_cube_with_slr_empty"); - assertTrue(ProjectManager.getInstance(this.getTestConfig()).getCubesByTable("default", "default.test_kylin_fact").contains(cube)); - - ProjectManager.getInstance(getTestConfig()).removeCubeFromProjects(cube.getName()); - - assertTrue(!ProjectManager.getInstance(this.getTestConfig()).getCubesByTable("default", "default.test_kylin_fact").contains(cube)); - - ProjectManager.getInstance(getTestConfig()).updateCubeToProject(cube.getName(), "default", "tester"); - - assertTrue(ProjectManager.getInstance(this.getTestConfig()).getCubesByTable("default", "default.test_kylin_fact").contains(cube)); - } - - private MetadataManager getMetadataManager() { - return MetadataManager.getInstance(getTestConfig()); - } - - private CubeManager getCubeManager() { - return CubeManager.getInstance(getTestConfig()); - } - - public CubeDescManager getCubeDescManager() { - return CubeDescManager.getInstance(getTestConfig()); - } -} diff --git a/job/src/test/java/com/kylinolap/job/engine/GenericJobEngineTest.java b/job/src/test/java/com/kylinolap/job/engine/GenericJobEngineTest.java index d7d1486..081bb92 100644 --- a/job/src/test/java/com/kylinolap/job/engine/GenericJobEngineTest.java +++ b/job/src/test/java/com/kylinolap/job/engine/GenericJobEngineTest.java @@ -23,6 +23,7 @@ import java.util.ArrayList; import java.util.List; +import com.kylinolap.cube.project.CubeRealizationManager; import org.apache.commons.io.FileUtils; import org.junit.AfterClass; import org.junit.Before; @@ -39,7 +40,6 @@ import com.kylinolap.cube.CubeSegment; import com.kylinolap.cube.CubeSegmentStatusEnum; import com.kylinolap.cube.exception.CubeIntegrityException; -import com.kylinolap.cube.project.ProjectManager; import com.kylinolap.job.JobDAO; import com.kylinolap.job.JobInstance; import com.kylinolap.job.JobInstance.JobStep; @@ -162,7 +162,7 @@ public void before() throws Exception { MetadataManager.removeInstance(KylinConfig.getInstanceFromEnv()); CubeManager.removeInstance(KylinConfig.getInstanceFromEnv()); - ProjectManager.removeInstance(KylinConfig.getInstanceFromEnv()); + CubeRealizationManager.removeInstance(KylinConfig.getInstanceFromEnv()); } @Test(expected = InvalidJobInstanceException.class) diff --git a/job/src/test/java/com/kylinolap/job/engine/JobInstanceBuilderTest.java b/job/src/test/java/com/kylinolap/job/engine/JobInstanceBuilderTest.java index 833e41a..f0d02c3 100644 --- a/job/src/test/java/com/kylinolap/job/engine/JobInstanceBuilderTest.java +++ b/job/src/test/java/com/kylinolap/job/engine/JobInstanceBuilderTest.java @@ -21,6 +21,7 @@ import java.util.TimeZone; import java.util.UUID; +import com.kylinolap.cube.project.CubeRealizationManager; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -32,7 +33,6 @@ import com.kylinolap.cube.CubeInstance; import com.kylinolap.cube.CubeManager; import com.kylinolap.cube.CubeSegment; -import com.kylinolap.cube.project.ProjectManager; import com.kylinolap.dict.DictionaryManager; import com.kylinolap.job.JobInstance; import com.kylinolap.job.JobInstance.JobStep; @@ -51,7 +51,7 @@ public void before() throws Exception { this.createTestMetadata(); MetadataManager.removeInstance(this.getTestConfig()); CubeManager.removeInstance(this.getTestConfig()); - ProjectManager.removeInstance(this.getTestConfig()); + CubeRealizationManager.removeInstance(this.getTestConfig()); DictionaryManager.removeInstance(this.getTestConfig()); } diff --git a/job/src/test/java/com/kylinolap/job/hadoop/cube/MergeCuboidMapperTest.java b/job/src/test/java/com/kylinolap/job/hadoop/cube/MergeCuboidMapperTest.java index bb072a1..8d844c4 100644 --- a/job/src/test/java/com/kylinolap/job/hadoop/cube/MergeCuboidMapperTest.java +++ b/job/src/test/java/com/kylinolap/job/hadoop/cube/MergeCuboidMapperTest.java @@ -22,6 +22,7 @@ import java.util.ArrayList; import java.util.List; +import com.kylinolap.cube.project.CubeRealizationManager; import org.apache.commons.io.FileUtils; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.Text; @@ -38,7 +39,6 @@ import com.kylinolap.cube.CubeManager; import com.kylinolap.cube.CubeSegment; import com.kylinolap.cube.exception.CubeIntegrityException; -import com.kylinolap.cube.project.ProjectManager; import com.kylinolap.dict.Dictionary; import com.kylinolap.dict.DictionaryGenerator; import com.kylinolap.dict.DictionaryInfo; @@ -93,7 +93,7 @@ public void setUp() throws Exception { MetadataManager.removeInstance(this.getTestConfig()); CubeManager.removeInstance(this.getTestConfig()); - ProjectManager.removeInstance(this.getTestConfig()); + CubeRealizationManager.removeInstance(this.getTestConfig()); DictionaryManager.removeInstance(this.getTestConfig()); // hack for distributed cache diff --git a/server/src/main/java/com/kylinolap/rest/controller/TableController.java b/server/src/main/java/com/kylinolap/rest/controller/TableController.java index 9b036aa..8a88497 100644 --- a/server/src/main/java/com/kylinolap/rest/controller/TableController.java +++ b/server/src/main/java/com/kylinolap/rest/controller/TableController.java @@ -70,7 +70,7 @@ long start = System.currentTimeMillis(); List tables = null; try { - tables = cubeMgmtService.getProjectManager().listDefinedTablesInProject(project); + tables = cubeMgmtService.getCubeRealizationManager().listDefinedTablesInProject(project); } catch (Exception e) { logger.error("Failed to deal with the request.", e); throw new InternalErrorException(e.getLocalizedMessage()); diff --git a/server/src/main/java/com/kylinolap/rest/service/BasicService.java b/server/src/main/java/com/kylinolap/rest/service/BasicService.java index 92b6cb7..8da2a99 100644 --- a/server/src/main/java/com/kylinolap/rest/service/BasicService.java +++ b/server/src/main/java/com/kylinolap/rest/service/BasicService.java @@ -33,6 +33,7 @@ import com.kylinolap.cube.project.CubeRealizationManager; import com.kylinolap.metadata.project.ProjectInstance; +import com.kylinolap.metadata.project.ProjectManager; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -157,7 +158,11 @@ public CubeDescManager getCubeDescManager() { return CubeDescManager.getInstance(getConfig()); } - public CubeRealizationManager getProjectManager() { + public ProjectManager getProjectManager() { + return ProjectManager.getInstance(getConfig()); + } + + public CubeRealizationManager getCubeRealizationManager() { return CubeRealizationManager.getInstance(getConfig()); } diff --git a/server/src/main/java/com/kylinolap/rest/service/CubeService.java b/server/src/main/java/com/kylinolap/rest/service/CubeService.java index de58372..8e49dee 100644 --- a/server/src/main/java/com/kylinolap/rest/service/CubeService.java +++ b/server/src/main/java/com/kylinolap/rest/service/CubeService.java @@ -31,7 +31,11 @@ import java.util.Map; import java.util.Set; +import com.kylinolap.cube.project.CubeRealizationManager; +import com.kylinolap.metadata.model.realization.DataModelRealizationType; +import com.kylinolap.metadata.project.ProjectDataModel; import com.kylinolap.metadata.project.ProjectInstance; +import com.kylinolap.metadata.project.ProjectManager; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; @@ -103,7 +107,7 @@ if (null == project) { cubeInstances = getCubeManager().listAllCubes(); } else { - cubeInstances = getProjectManager().listAllCubes(projectName); + cubeInstances = listAllCubes(projectName); } List filterCubes = new ArrayList(); @@ -187,6 +191,41 @@ public CubeInstance createCubeAndDesc(String cubeName, String projectName, CubeD return createdCube; } + private List listAllCubes(String projectName) { + ProjectManager projectManager = getProjectManager(); + ProjectInstance project = projectManager.getProject(projectName); + if (project == null) { + return Collections.emptyList(); + } + ArrayList result = new ArrayList(); + for (ProjectDataModel projectDataModel: project.getDataModels()) { + if (projectDataModel.getType() == DataModelRealizationType.CUBE) { + CubeInstance cube = getCubeManager().getCube(projectDataModel.getRealization()); + assert cube != null; + result.add(cube); + } + } + return result; + } + + private boolean isCubeInProject(String projectName, CubeInstance target) { + ProjectManager projectManager = getProjectManager(); + ProjectInstance project = projectManager.getProject(projectName); + if (project == null) { + return false; + } + for (ProjectDataModel projectDataModel: project.getDataModels()) { + if (projectDataModel.getType() == DataModelRealizationType.CUBE) { + CubeInstance cube = getCubeManager().getCube(projectDataModel.getRealization()); + assert cube != null; + if (cube.equals(target)) { + return true; + } + } + } + return false; + } + @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN + " or hasPermission(#cube, 'ADMINISTRATION') or hasPermission(#cube, 'MANAGEMENT')") public CubeDesc updateCubeAndDesc(CubeInstance cube, CubeDesc desc, String newProjectName) throws UnknownHostException, IOException, JobException { List jobInstances = this.getJobManager().listJobs(cube.getName(), null); @@ -205,10 +244,11 @@ public CubeDesc updateCubeAndDesc(CubeInstance cube, CubeDesc desc, String newPr int cuboidCount = CuboidCLI.simulateCuboidGeneration(updatedCubeDesc); logger.info("Updated cube " + cube.getName() + " has " + cuboidCount + " cuboids"); - - if (!getProjectManager().isCubeInProject(newProjectName, cube)) { + + ProjectManager projectManager = getProjectManager(); + if (!isCubeInProject(newProjectName, cube)) { String owner = SecurityContextHolder.getContext().getAuthentication().getName(); - ProjectInstance newProject = getProjectManager().updateCubeToProject(cube.getName(), newProjectName, owner); + ProjectInstance newProject = projectManager.updateCubeToProject(cube.getName(), newProjectName, owner); accessService.inherit(cube, newProject); } @@ -597,6 +637,6 @@ private void releaseAllSegments(CubeInstance cube) throws IOException, JobExcept @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN) public void syncTableToProject(String tables,String project) throws IOException { - getProjectManager().updateTableToProject(tables, project); + getCubeRealizationManager().updateTableToProject(tables, project); } } diff --git a/server/src/main/java/com/kylinolap/rest/service/QueryService.java b/server/src/main/java/com/kylinolap/rest/service/QueryService.java index badad90..d2b147f 100644 --- a/server/src/main/java/com/kylinolap/rest/service/QueryService.java +++ b/server/src/main/java/com/kylinolap/rest/service/QueryService.java @@ -247,10 +247,6 @@ public void logQuery(final SQLRequest request, final SQLResponse response, final logger.info(stringBuilder.toString()); } - /** - * @param sql - * @throws SQLException - */ @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN + " or hasPermission(#cube, 'ADMINISTRATION') or hasPermission(#cube, 'MANAGEMENT')" + " or hasPermission(#cube, 'OPERATION') or hasPermission(#cube, 'READ')") public void checkAuthorization(CubeInstance cube) throws AccessDeniedException { } @@ -301,7 +297,7 @@ protected SQLResponse executeQuery(String sql, SQLRequest sqlRequest) throws Exc // PostgreSQL has only 5 TableMeta tblMeta = new TableMeta(catalogName == null ? Constant.FakeCatalogName : catalogName, schemaName == null ? Constant.FakeSchemaName : schemaName, JDBCTableMeta.getString(3), JDBCTableMeta.getString(4), JDBCTableMeta.getString(5), null, null, null, null, null); - if (!cubedOnly || getProjectManager().isExposedTable(project, schemaName + "." + tblMeta.getTABLE_NAME())) { + if (!cubedOnly || getCubeRealizationManager().isExposedTable(project, schemaName + "." + tblMeta.getTABLE_NAME())) { tableMetas.add(tblMeta); tableMap.put(tblMeta.getTABLE_SCHEM() + "#" + tblMeta.getTABLE_NAME(), tblMeta); } @@ -317,7 +313,7 @@ protected SQLResponse executeQuery(String sql, SQLRequest sqlRequest) throws Exc // kylin(optiq) is not strictly following JDBC specification ColumnMeta colmnMeta = new ColumnMeta(catalogName == null ? Constant.FakeCatalogName : catalogName, schemaName == null ? Constant.FakeSchemaName : schemaName, columnMeta.getString(3), columnMeta.getString(4), columnMeta.getInt(5), columnMeta.getString(6), columnMeta.getInt(7), getInt(columnMeta.getString(8)), columnMeta.getInt(9), columnMeta.getInt(10), columnMeta.getInt(11), columnMeta.getString(12), columnMeta.getString(13), getInt(columnMeta.getString(14)), getInt(columnMeta.getString(15)), columnMeta.getInt(16), columnMeta.getInt(17), columnMeta.getString(18), columnMeta.getString(19), columnMeta.getString(20), columnMeta.getString(21), getShort(columnMeta.getString(22)), columnMeta.getString(23)); - if (!cubedOnly || getProjectManager().isExposedColumn(project, schemaName + "." +colmnMeta.getTABLE_NAME(), colmnMeta.getCOLUMN_NAME())) { + if (!cubedOnly || getCubeRealizationManager().isExposedColumn(project, schemaName + "." +colmnMeta.getTABLE_NAME(), colmnMeta.getCOLUMN_NAME())) { tableMap.get(colmnMeta.getTABLE_SCHEM() + "#" + colmnMeta.getTABLE_NAME()).addColumn(colmnMeta); } } @@ -331,7 +327,7 @@ protected SQLResponse executeQuery(String sql, SQLRequest sqlRequest) throws Exc /** * @param sql - * @param project + * @param sqlRequest * @return * @throws Exception */ diff --git a/server/src/test/java/com/kylinolap/rest/controller/ProjectControllerTest.java b/server/src/test/java/com/kylinolap/rest/controller/ProjectControllerTest.java index de25bc9..6dd31b4 100644 --- a/server/src/test/java/com/kylinolap/rest/controller/ProjectControllerTest.java +++ b/server/src/test/java/com/kylinolap/rest/controller/ProjectControllerTest.java @@ -3,13 +3,13 @@ import java.io.IOException; import java.util.List; +import com.kylinolap.metadata.project.ProjectManager; import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; import com.kylinolap.metadata.project.ProjectInstance; -import com.kylinolap.cube.project.ProjectManager; import com.kylinolap.rest.exception.InternalErrorException; import com.kylinolap.rest.request.CreateProjectRequest; import com.kylinolap.rest.request.UpdateProjectRequest; diff --git a/server/src/test/java/com/kylinolap/rest/service/ServiceTestBase.java b/server/src/test/java/com/kylinolap/rest/service/ServiceTestBase.java index 2d4229f..dbbf3fc 100644 --- a/server/src/test/java/com/kylinolap/rest/service/ServiceTestBase.java +++ b/server/src/test/java/com/kylinolap/rest/service/ServiceTestBase.java @@ -16,6 +16,7 @@ package com.kylinolap.rest.service; +import com.kylinolap.cube.project.CubeRealizationManager; import org.junit.After; import org.junit.Before; import org.junit.BeforeClass; @@ -31,7 +32,6 @@ import com.kylinolap.common.KylinConfig; import com.kylinolap.common.util.HBaseMetadataTestCase; import com.kylinolap.cube.CubeManager; -import com.kylinolap.cube.project.ProjectManager; import com.kylinolap.metadata.MetadataManager; /** @@ -64,7 +64,7 @@ public void setUp() { MetadataManager.removeInstance(getTestConfig()); CubeManager.removeInstance(this.getTestConfig()); - ProjectManager.removeInstance(this.getTestConfig()); + CubeRealizationManager.removeInstance(this.getTestConfig()); } @After