From d5978b07d276799ecd66be1e67c290d0813ce361 Mon Sep 17 00:00:00 2001 From: "qianhao.zhou" Date: Tue, 9 Dec 2014 11:38:06 +0800 Subject: [PATCH] fix CI issue --- .../main/java/com/kylinolap/cube/CubeManager.java | 3 + .../cube/project/CubeRealizationManager.java | 260 ++++----------------- .../cube/project/CubeRealizationManagerTest.java | 14 +- .../metadata/project/ProjectInstance.java | 18 +- .../kylinolap/metadata/project/ProjectManager.java | 13 +- .../com/kylinolap/rest/service/CubeService.java | 4 +- .../com/kylinolap/rest/service/ProjectService.java | 2 + 7 files changed, 79 insertions(+), 235 deletions(-) diff --git a/cube/src/main/java/com/kylinolap/cube/CubeManager.java b/cube/src/main/java/com/kylinolap/cube/CubeManager.java index 1302c5d..140d61f 100644 --- a/cube/src/main/java/com/kylinolap/cube/CubeManager.java +++ b/cube/src/main/java/com/kylinolap/cube/CubeManager.java @@ -27,6 +27,7 @@ import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; +import com.kylinolap.cube.project.CubeRealizationManager; import com.kylinolap.metadata.project.ProjectInstance; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; @@ -239,6 +240,7 @@ public CubeInstance dropCube(String cubeName, boolean deleteDesc) throws IOExcep // delete cube from project ProjectManager.getInstance(config).removeCubeFromProjects(cubeName); + CubeRealizationManager.getInstance(config).loadAllProjects(); // clean cube cache this.afterCubeDroped(cube, projects); @@ -256,6 +258,7 @@ public CubeInstance createCube(String cubeName, String projectName, CubeDesc des saveResource(cube); ProjectManager.getInstance(config).updateCubeToProject(cubeName, projectName, owner); + CubeRealizationManager.getInstance(config).loadProject(ProjectManager.getInstance(config).getProject(projectName)); return cube; } diff --git a/cube/src/main/java/com/kylinolap/cube/project/CubeRealizationManager.java b/cube/src/main/java/com/kylinolap/cube/project/CubeRealizationManager.java index a43ca67..dc88549 100644 --- a/cube/src/main/java/com/kylinolap/cube/project/CubeRealizationManager.java +++ b/cube/src/main/java/com/kylinolap/cube/project/CubeRealizationManager.java @@ -21,6 +21,7 @@ import java.util.concurrent.ConcurrentHashMap; import com.kylinolap.metadata.project.ProjectInstance; +import com.kylinolap.metadata.project.ProjectManager; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -30,11 +31,6 @@ import com.google.common.collect.Multimap; import com.google.common.collect.Multimaps; import com.kylinolap.common.KylinConfig; -import com.kylinolap.common.persistence.JsonSerializer; -import com.kylinolap.common.persistence.ResourceStore; -import com.kylinolap.common.persistence.Serializer; -import com.kylinolap.common.restclient.Broadcaster; -import com.kylinolap.common.restclient.SingleValueCache; import com.kylinolap.cube.CubeInstance; import com.kylinolap.cube.CubeManager; import com.kylinolap.cube.model.CubeDesc; @@ -55,14 +51,18 @@ // static cached instances private static final ConcurrentHashMap CACHE = new ConcurrentHashMap(); - private static final Serializer PROJECT_SERIALIZER = new JsonSerializer(ProjectInstance.class); private KylinConfig config; - // project name => ProjrectDesc - private SingleValueCache projectMap = new SingleValueCache(Broadcaster.TYPE.PROJECT); // project name => tables private Multimap projectTables = Multimaps.synchronizedMultimap(HashMultimap.create()); + private CubeRealizationManager(KylinConfig config) throws IOException { + logger.info("Initializing CubeManager with metadata url " + config); + this.config = config; + + loadAllProjects(); + } + public static CubeRealizationManager getInstance(KylinConfig config) { CubeRealizationManager r = CACHE.get(config); if (r != null) { @@ -91,102 +91,11 @@ public static synchronized void removeInstance(KylinConfig config) { CACHE.remove(config); } - private CubeRealizationManager(KylinConfig config) throws IOException { - logger.info("Initializing CubeManager with metadata url " + config); - this.config = config; - - loadAllProjects(); - } - - public static String getDefaultProjectName() { - return ProjectInstance.DEFAULT_PROJECT_NAME; - } - - public List listAllProjects() { - return new ArrayList(projectMap.values()); - } - - public List getProjects(String cubeName) { - return this.findProjects(cubeName); - } - - public ProjectInstance dropProject(String projectName) throws IOException { - if (projectName == null) - throw new IllegalArgumentException("Project name not given"); - - ProjectInstance projectInstance = getProject(projectName); - - if (projectInstance == null) { - throw new IllegalStateException("The project named " + projectName + " does not exist"); - } - - if (projectInstance.getCubes().size() != 0) { - throw new IllegalStateException("The project named " + projectName + " can not be deleted because there's still cubes in it. Delete all the cubes first."); - } - - logger.info("Dropping project '" + projectInstance.getName() + "'"); - deleteResource(projectInstance); - - return projectInstance; - } - - public ProjectInstance getProject(String projectName) { - if (projectName == null) - return null; - projectName = ProjectInstance.getNormalizedProjectName(projectName); - return projectMap.get(projectName); - } - - public ProjectInstance createProject(String projectName, String owner, String description) throws IOException { - - logger.info("Creating project '" + projectName); - - ProjectInstance currentProject = getProject(projectName); - if (currentProject == null) { - currentProject = ProjectInstance.create(projectName, owner, description, null); - } else { - throw new IllegalStateException("The project named " + projectName + "already exists"); - } - - saveResource(currentProject); - - return currentProject; - } - - public ProjectInstance updateProject(ProjectInstance project, String newName, String newDesc) throws IOException { - if (!project.getName().equals(newName)) { - ProjectInstance newProject = this.createProject(newName, project.getOwner(), newDesc); - newProject.setCreateTime(project.getCreateTime()); - newProject.recordUpdateTime(System.currentTimeMillis()); - newProject.setCubes(project.getCubes()); - - deleteResource(project); - saveResource(newProject); - - return newProject; - } else { - project.setName(newName); - project.setDescription(newDesc); - - if (project.getUuid() == null) - project.updateRandomUuid(); - - saveResource(project); - - return project; - } - } - public boolean isCubeInProject(String projectName, CubeInstance cube) { return this.listAllCubes(projectName).contains(cube); } -// public ProjectInstance updateCubeToProject(String cubeName, String newProjectName, String owner) throws IOException { -// removeCubeFromProjects(cubeName); -// return addCubeToProject(cubeName, newProjectName, owner); -// } - - public ProjectInstance updateTableToProject(String tables, String projectName) throws IOException { + public ProjectInstance addTablesToProject(String tables, String projectName) throws IOException { ProjectInstance projectInstance = getProject(projectName); String[] tokens = StringUtils.split(tables, ","); for (int i = 0; i < tokens.length; i++) { @@ -205,15 +114,6 @@ public ProjectInstance updateTableToProject(String tables, String projectName) t return projectInstance; } - -// public void removeCubeFromProjects(String cubeName) throws IOException { -// for (ProjectInstance projectInstance : findProjects(cubeName)) { -// projectInstance.removeCube(cubeName); -// -// saveResource(projectInstance); -// } -// } - public List listExposedTables(String project) { project = ProjectInstance.getNormalizedProjectName(project); List tables = Lists.newArrayList(); @@ -230,7 +130,7 @@ public ProjectInstance updateTableToProject(String tables, String projectName) t public List listDefinedTablesInProject(String project) throws IOException { - if(null==project){ + if(null == project){ return Collections.emptyList(); } project = ProjectInstance.getNormalizedProjectName(project); @@ -283,23 +183,23 @@ public boolean isExposedColumn(String project, String table, String col) { return getProjectTable(project, table).getColumns().contains(col); } - public List listAllCubes(String project) { - project = ProjectInstance.getNormalizedProjectName(project); + public List listAllCubes(String projectName) { + return listAllCubes(ProjectManager.getInstance(config).getProject(projectName)); + } + public List listAllCubes(ProjectInstance projectInstance) { + if (projectInstance == null) { + return Collections.emptyList(); + } HashSet ret = new HashSet(); - - ProjectInstance projectInstance = getProject(project); - if (projectInstance != null) { - for (String cubeName : projectInstance.getCubes()) { - CubeInstance cube = CubeManager.getInstance(config).getCube(cubeName); - if (null != cube) { - ret.add(cube); - } else { - logger.error("Failed to load cube " + cubeName); - } + for (String cubeName : projectInstance.getCubes()) { + CubeInstance cube = CubeManager.getInstance(config).getCube(cubeName); + if (null != cube) { + ret.add(cube); + } else { + logger.error("Failed to load cube " + cubeName); } } - return new ArrayList(ret); } @@ -354,19 +254,26 @@ public boolean isExposedColumn(String project, String table, String col) { return result; } - public void loadProjectCache(ProjectInstance project, boolean triggerUpdate) throws IOException { - loadProject(project.getResourcePath(), triggerUpdate); - loadTables(project.getResourcePath()); - } - - public void removeProjectCache(ProjectInstance project) { + public void unloadProject(ProjectInstance project) { String projectName = ProjectInstance.getNormalizedProjectName(project.getName()); - if (projectMap.containsKey(projectName)) { - projectMap.remove(projectName); + if (projectTables.containsKey(projectName)) { projectTables.removeAll(projectName); } } + public void loadProject(ProjectInstance project) throws IOException { + loadTables(project); + } + + public final void loadAllProjects() throws IOException { + List projectInstances = ProjectManager.getInstance(config).listAllProjects(); + for (ProjectInstance projectInstance : projectInstances) { + loadTables(projectInstance); + } + + logger.debug("Loaded " + projectInstances.size() + " Project(s)"); + } + private void mapTableToCube(ProjectInstance projectInstance, CubeInstance cubeInstance) { // schema sanity check CubeDesc cubeDesc = cubeInstance.getDescriptor(); @@ -398,9 +305,9 @@ private void mapTableToCube(ProjectInstance projectInstance, CubeInstance cubeIn } } - private List findProjects(String cubeName) { + private List findProjectsByCubeName(String cubeName) { List projects = new ArrayList(); - for (ProjectInstance projectInstance : projectMap.values()) { + for (ProjectInstance projectInstance : getProjectManager().listAllProjects()) { if (projectInstance.containsCube(cubeName)) { projects.add(projectInstance); } @@ -409,33 +316,7 @@ private void mapTableToCube(ProjectInstance projectInstance, CubeInstance cubeIn return projects; } - private synchronized ProjectInstance loadProject(String path, boolean triggerUpdate) throws IOException { - ResourceStore store = getStore(); - logger.debug("Loading CubeInstance " + store.getReadableResourcePath(path)); - - ProjectInstance projectInstance = store.getResource(path, ProjectInstance.class, PROJECT_SERIALIZER); - projectInstance.init(); - - if (StringUtils.isBlank(projectInstance.getName())) { - throw new IllegalStateException("Project name must not be blank"); - } - - if (triggerUpdate) { - projectMap.put(projectInstance.getName().toUpperCase(), projectInstance); - } else { - projectMap.putLocal(projectInstance.getName().toUpperCase(), projectInstance); - } - - return projectInstance; - } - - private synchronized void loadTables(String path) throws IOException { - ResourceStore store = getStore(); - logger.debug("Loading CubeInstance " + store.getReadableResourcePath(path)); - - ProjectInstance projectInstance = store.getResource(path, ProjectInstance.class, PROJECT_SERIALIZER); - projectInstance.init(); - + private synchronized void loadTables(ProjectInstance projectInstance) throws IOException { String project = ProjectInstance.getNormalizedProjectName(projectInstance.getName()); projectTables.removeAll(project); @@ -445,54 +326,8 @@ private synchronized void loadTables(String path) throws IOException { } } - private void loadAllProjects() throws IOException { - ResourceStore store = getStore(); - List paths = store.collectResourceRecursively(ResourceStore.PROJECT_RESOURCE_ROOT, ".json"); - - logger.debug("Loading Project from folder " + store.getReadableResourcePath(ResourceStore.PROJECT_RESOURCE_ROOT)); - - for (String path : paths) { - loadProject(path, false); - loadTables(path); - } - - logger.debug("Loaded " + paths.size() + " Project(s)"); - } - - private ProjectInstance addCubeToProject(String cubeName, String project, String user) throws IOException { - String newProjectName = ProjectInstance.getNormalizedProjectName(project); - ProjectInstance newProject = getProject(newProjectName); - if (newProject == null) { - newProject = this.createProject(newProjectName, user, "This is a project automatically added when adding cube " + cubeName); - } - newProject.addCube(cubeName); - saveResource(newProject); - - return newProject; - } - - private void saveResource(ProjectInstance proj) throws IOException { - ResourceStore store = getStore(); - store.putResource(proj.getResourcePath(), proj, PROJECT_SERIALIZER); - afterProjectUpdated(proj); - } - - private void deleteResource(ProjectInstance proj) throws IOException { - ResourceStore store = getStore(); - store.deleteResource(proj.getResourcePath()); - this.afterProjectDropped(proj); - } - - private void afterProjectUpdated(ProjectInstance updatedProject) { - try { - this.loadProjectCache(updatedProject, true); - } catch (IOException e) { - logger.error(e.getLocalizedMessage(), e); - } - } - - private void afterProjectDropped(ProjectInstance droppedProject) { - this.removeProjectCache(droppedProject); + private ProjectInstance getProject(String projectName) { + return ProjectManager.getInstance(config).getProject(projectName); } // sync on update @@ -573,11 +408,16 @@ private ProjectTable getProjectTable(String project, final String table, boolean return projectTable; } - private ResourceStore getStore() { - return ResourceStore.getStore(this.config); + private void saveResource(ProjectInstance projectInstance) throws IOException { + getProjectManager().updateProject(projectInstance); } private MetadataManager getMetadataManager() { return MetadataManager.getInstance(config); } + + private ProjectManager getProjectManager() { + return ProjectManager.getInstance(config); + } + } diff --git a/cube/src/test/java/com/kylinolap/cube/project/CubeRealizationManagerTest.java b/cube/src/test/java/com/kylinolap/cube/project/CubeRealizationManagerTest.java index 90b2efd..ceeb4ce 100644 --- a/cube/src/test/java/com/kylinolap/cube/project/CubeRealizationManagerTest.java +++ b/cube/src/test/java/com/kylinolap/cube/project/CubeRealizationManagerTest.java @@ -56,12 +56,14 @@ public void after() throws Exception { @Test(expected = IllegalStateException.class) public void testDropNonemptyProject1() throws IOException { - ProjectManager.getInstance(this.getTestConfig()).dropProject("DEFAULT"); + ProjectInstance project = ProjectManager.getInstance(this.getTestConfig()).dropProject("DEFAULT"); + CubeRealizationManager.getInstance(this.getTestConfig()).unloadProject(project); } @Test(expected = IllegalStateException.class) public void testDropNonemptyProject2() throws IOException { - ProjectManager.getInstance(this.getTestConfig()).dropProject("DEFAULT???"); + ProjectInstance project = ProjectManager.getInstance(this.getTestConfig()).dropProject("DEFAULT???"); + CubeRealizationManager.getInstance(this.getTestConfig()).unloadProject(project); } @Test @@ -89,11 +91,13 @@ public void testNewProject() throws Exception { assertTrue(CubeManager.getInstance(this.getTestConfig()).listAllCubes().size() == originalCubeCount + 1); projectmanager.updateCubeToProject("cube_in_alien_project", "default", null); + CubeRealizationManager.getInstance(getTestConfig()).loadProject(projectmanager.getProject("default")); assertTrue(cubeRealizationManager.listAllCubes("ALIEN").size() == 0); assertTrue(cubeRealizationManager.listAllCubes("default").size() == originalCubeCountInDefault + 1); assertTrue(cubeRealizationManager.listAllCubes("default").contains(createdCube)); projectmanager.updateCubeToProject("cube_in_alien_project", "alien", null); + CubeRealizationManager.getInstance(getTestConfig()).loadProject(projectmanager.getProject("default")); assertTrue(cubeRealizationManager.listAllCubes("ALIEN").size() == 1); assertTrue(cubeRealizationManager.listAllCubes("default").size() == originalCubeCountInDefault); assertTrue(cubeRealizationManager.listAllCubes("alien").contains(createdCube)); @@ -107,7 +111,8 @@ public void testNewProject() throws Exception { assertTrue(projectmanager.listAllProjects().size() == originalProjectCount + 1); assertTrue(CubeManager.getInstance(this.getTestConfig()).listAllCubes().size() == originalCubeCount); - projectmanager.dropProject("alien"); + ProjectInstance project = projectmanager.dropProject("alien"); + CubeRealizationManager.getInstance(this.getTestConfig()).unloadProject(project); assertTrue(projectmanager.listAllProjects().size() == originalProjectCount); } @@ -158,10 +163,13 @@ public void testProjectsLoadAfterProjectChange() throws IOException { assertTrue(cubeRealizationManager.getCubesByTable("default", "default.test_kylin_fact").contains(cube)); ProjectManager.getInstance(getTestConfig()).removeCubeFromProjects(cube.getName()); + CubeRealizationManager.getInstance(getTestConfig()).loadAllProjects(); + assertTrue(!cubeRealizationManager.getCubesByTable("default", "default.test_kylin_fact").contains(cube)); ProjectManager.getInstance(getTestConfig()).updateCubeToProject(cube.getName(), "default", "tester"); + CubeRealizationManager.getInstance(getTestConfig()).loadProject(ProjectManager.getInstance(getTestConfig()).getProject("default")); assertTrue(cubeRealizationManager.getCubesByTable("default", "default.test_kylin_fact").contains(cube)); } diff --git a/metadata/src/main/java/com/kylinolap/metadata/project/ProjectInstance.java b/metadata/src/main/java/com/kylinolap/metadata/project/ProjectInstance.java index bb0ae33..cfdc54b 100644 --- a/metadata/src/main/java/com/kylinolap/metadata/project/ProjectInstance.java +++ b/metadata/src/main/java/com/kylinolap/metadata/project/ProjectInstance.java @@ -144,13 +144,11 @@ public void setName(String name) { } public boolean containsCube(String cubeName) { - cubeName = cubeName.toUpperCase(); - return cubes.contains(cubeName); + return cubes.contains(cubeName.toUpperCase()); } public void removeCube(String cubeName) { - cubeName = cubeName.toUpperCase(); - cubes.remove(cubeName); + cubes.remove(cubeName.toUpperCase()); } public int getCubesCount() { @@ -158,8 +156,7 @@ public int getCubesCount() { } public void addCube(String cubeName) { - cubeName = cubeName.toUpperCase(); - this.cubes.add(cubeName); + this.cubes.add(cubeName.toUpperCase()); } public List getCubes() { @@ -176,13 +173,11 @@ public void setTables(Set tables) { } public boolean containsTable(String tableName) { - tableName = tableName.toUpperCase(); - return tables.contains(tableName); + return tables.contains(tableName.toUpperCase()); } public void removeTable(String tableName) { - tableName = tableName.toUpperCase(); - tables.remove(tableName); + tables.remove(tableName.toUpperCase()); } public int getTablesCount() { @@ -190,8 +185,7 @@ public int getTablesCount() { } public void addTable(String tableName) { - tableName = tableName.toUpperCase(); - this.getTables().add(tableName); + this.getTables().add(tableName.toUpperCase()); } //will return new Set for null diff --git a/metadata/src/main/java/com/kylinolap/metadata/project/ProjectManager.java b/metadata/src/main/java/com/kylinolap/metadata/project/ProjectManager.java index 42bbefa..ba463f8 100644 --- a/metadata/src/main/java/com/kylinolap/metadata/project/ProjectManager.java +++ b/metadata/src/main/java/com/kylinolap/metadata/project/ProjectManager.java @@ -169,6 +169,11 @@ public ProjectInstance updateProject(ProjectInstance project, String newName, St } } + public ProjectInstance updateProject(ProjectInstance project) throws IOException{ + saveResource(project); + return project; + } + public void removeCubeFromProjects(String cubeName) throws IOException { for (ProjectInstance projectInstance : findProjects(cubeName)) { projectInstance.removeCube(cubeName); @@ -181,14 +186,6 @@ public ProjectInstance updateCubeToProject(String cubeName, String newProjectNam return addCubeToProject(cubeName, newProjectName, owner); } - public boolean containsRealization(String projectName, DataModelRealizationType realization, String name) { - ProjectInstance project = getProject(projectName); - if (project == null) { - return false; - } - return project.containsRealization(realization, name); - } - public void loadProjectCache(ProjectInstance project, boolean triggerUpdate) throws IOException { loadProject(project.getResourcePath(), triggerUpdate); } diff --git a/server/src/main/java/com/kylinolap/rest/service/CubeService.java b/server/src/main/java/com/kylinolap/rest/service/CubeService.java index 8e49dee..01ed3d0 100644 --- a/server/src/main/java/com/kylinolap/rest/service/CubeService.java +++ b/server/src/main/java/com/kylinolap/rest/service/CubeService.java @@ -31,7 +31,6 @@ import java.util.Map; import java.util.Set; -import com.kylinolap.cube.project.CubeRealizationManager; import com.kylinolap.metadata.model.realization.DataModelRealizationType; import com.kylinolap.metadata.project.ProjectDataModel; import com.kylinolap.metadata.project.ProjectInstance; @@ -249,6 +248,7 @@ public CubeDesc updateCubeAndDesc(CubeInstance cube, CubeDesc desc, String newPr if (!isCubeInProject(newProjectName, cube)) { String owner = SecurityContextHolder.getContext().getAuthentication().getName(); ProjectInstance newProject = projectManager.updateCubeToProject(cube.getName(), newProjectName, owner); + getCubeRealizationManager().loadProject(getProjectManager().getProject(newProjectName)); accessService.inherit(cube, newProject); } @@ -637,6 +637,6 @@ private void releaseAllSegments(CubeInstance cube) throws IOException, JobExcept @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN) public void syncTableToProject(String tables,String project) throws IOException { - getCubeRealizationManager().updateTableToProject(tables, project); + getCubeRealizationManager().addTablesToProject(tables, project); } } diff --git a/server/src/main/java/com/kylinolap/rest/service/ProjectService.java b/server/src/main/java/com/kylinolap/rest/service/ProjectService.java index a170167..db1691b 100644 --- a/server/src/main/java/com/kylinolap/rest/service/ProjectService.java +++ b/server/src/main/java/com/kylinolap/rest/service/ProjectService.java @@ -56,6 +56,7 @@ public ProjectInstance createProject(CreateProjectRequest projectRequest) throws } String owner = SecurityContextHolder.getContext().getAuthentication().getName(); ProjectInstance createdProject = getProjectManager().createProject(projectName, owner, description); + getCubeRealizationManager().loadProject(createdProject); accessService.init(createdProject, AclPermission.ADMINISTRATION); logger.debug("New project created."); @@ -102,6 +103,7 @@ public ProjectInstance updateProject(UpdateProjectRequest projectRequest) throws public void deleteProject(String projectName) throws IOException { ProjectInstance project = getProjectManager().getProject(projectName); getProjectManager().dropProject(projectName); + getCubeRealizationManager().unloadProject(project); accessService.clean(project, true); }