From cd84abee0452bc0c348b7cee8601537609537e4f Mon Sep 17 00:00:00 2001 From: kangkaisen Date: Mon, 25 Dec 2017 18:33:31 +0800 Subject: [PATCH] KYLIN-2672 Only clean necessary cache for CubeMigrationCLI --- .../apache/kylin/common/restclient/RestClient.java | 20 +++++++ .../java/org/apache/kylin/cube/CubeManager.java | 2 +- .../kylin/metadata/TableMetadataManager.java | 12 +++++ .../org/apache/kylin/metadata/model/TableDesc.java | 2 +- .../kylin/metadata/project/ProjectL2Cache.java | 4 ++ .../kylin/metadata/project/ProjectManager.java | 6 ++- .../kylin/rest/controller/CacheController.java | 10 ++++ .../kylin/rest/request/CubeMigrationRequest.java | 62 ++++++++++++++++++++++ .../apache/kylin/rest/service/CacheService.java | 34 +++++++++++- .../org/apache/kylin/tool/CubeMigrationCLI.java | 18 ++++--- 10 files changed, 159 insertions(+), 11 deletions(-) create mode 100644 server-base/src/main/java/org/apache/kylin/rest/request/CubeMigrationRequest.java diff --git a/core-common/src/main/java/org/apache/kylin/common/restclient/RestClient.java b/core-common/src/main/java/org/apache/kylin/common/restclient/RestClient.java index 02045ae11..e1cc13c21 100644 --- a/core-common/src/main/java/org/apache/kylin/common/restclient/RestClient.java +++ b/core-common/src/main/java/org/apache/kylin/common/restclient/RestClient.java @@ -274,6 +274,26 @@ public class RestClient { return response; } + public void clearCacheForCubeMigration(String cube, String project, String model, Map tableToProjects) throws IOException{ + String url = baseUrl + "/cache/migration"; + HttpPost post = new HttpPost(url); + + post.addHeader("Accept", "application/json, text/plain, */*"); + post.addHeader("Content-Type", "application/json"); + + HashMap paraMap = new HashMap(); + paraMap.put("cube", cube); + paraMap.put("project", project); + paraMap.put("model", model); + paraMap.put("tableToProjects", tableToProjects); + String jsonMsg = JsonUtil.writeValueAsString(paraMap); + post.setEntity(new StringEntity(jsonMsg, "UTF-8")); + HttpResponse response = client.execute(post); + if (response.getStatusLine().getStatusCode() != 200) { + throw new IOException("Invalid response " + response.getStatusLine().getStatusCode()); + } + } + private HashMap dealResponse(HttpResponse response) throws IOException { if (response.getStatusLine().getStatusCode() != 200) { throw new IOException("Invalid response " + response.getStatusLine().getStatusCode()); diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java index df1d95e4d..99ffc523d 100755 --- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java @@ -386,7 +386,7 @@ public class CubeManager implements IRealizationProvider { } // for internal - CubeInstance reloadCubeQuietly(String cubeName) { + public CubeInstance reloadCubeQuietly(String cubeName) { try (AutoLock lock = cubeMapLock.lockForWrite()) { CubeInstance cube = crud.reloadQuietly(cubeName); if (cube != null) diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/TableMetadataManager.java b/core-metadata/src/main/java/org/apache/kylin/metadata/TableMetadataManager.java index f09c47c72..2308df495 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/TableMetadataManager.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/TableMetadataManager.java @@ -151,6 +151,12 @@ public class TableMetadataManager { } } + public void reloadSourceTable(String table, String project) { + try (AutoLock lock = srcTableMapLock.lockForWrite()) { + srcTableCrud.reloadQuietly(TableDesc.concatResourcePath(table, project)); + } + } + public List listAllTables(String prj) { try (AutoLock lock = srcTableMapLock.lockForWrite()) { return Lists.newArrayList(getAllTablesMap(prj).values()); @@ -314,6 +320,12 @@ public class TableMetadataManager { } } + public void reloadTableExt(String table, String project) { + try (AutoLock lock = srcExtMapLock.lockForWrite()) { + srcExtCrud.reloadQuietly(TableExtDesc.concatResourcePath(table, project)); + } + } + /** * Get table extended info. Keys are defined in {@link MetadataConstants} * diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java index 68bc5e9aa..be278de73 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java @@ -57,7 +57,7 @@ public class TableDesc extends RootPersistentEntity implements ISourceAware { // this method should only used for getting dest path when copying from src to dest. // if you want to get table's src path, use getResourcePath() instead. - private static String concatResourcePath(String tableIdentity, String prj) { + public static String concatResourcePath(String tableIdentity, String prj) { return concatRawResourcePath(makeResourceName(tableIdentity, prj)); } diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java b/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java index 70b6a1233..6e09ae8eb 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java @@ -187,6 +187,10 @@ class ProjectL2Cache { return result; } + public void reloadCacheByProject(String project) { + projectCaches.put(project, loadCache(project)); + } + private ProjectCache loadCache(String project) { logger.debug("Loading L2 project cache for " + project); ProjectCache projectCache = new ProjectCache(project); diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectManager.java b/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectManager.java index 0dd364d12..aae692d79 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectManager.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectManager.java @@ -117,10 +117,14 @@ public class ProjectManager { l2Cache.clear(); } + public void reloadProjectL2Cache(String project) { + l2Cache.reloadCacheByProject(project); + } + public ProjectInstance reloadProjectQuietly(String project) throws IOException { try (AutoLock lock = prjMapLock.lockForWrite()) { ProjectInstance prj = crud.reloadQuietly(project); - clearL2Cache(); + reloadProjectL2Cache(project); return prj; } } diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/CacheController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/CacheController.java index 992094b38..08b7cc4b3 100644 --- a/server-base/src/main/java/org/apache/kylin/rest/controller/CacheController.java +++ b/server-base/src/main/java/org/apache/kylin/rest/controller/CacheController.java @@ -22,6 +22,7 @@ import java.io.IOException; import org.apache.kylin.common.KylinConfig; import org.apache.kylin.metadata.cachesync.Broadcaster; +import org.apache.kylin.rest.request.CubeMigrationRequest; import org.apache.kylin.rest.service.CacheService; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -29,6 +30,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Controller; import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestMethod; import org.springframework.web.bind.annotation.ResponseBody; @@ -73,6 +75,14 @@ public class CacheController extends BasicController { cacheService.notifyMetadataChange(Broadcaster.SYNC_ALL, Broadcaster.Event.UPDATE, Broadcaster.SYNC_ALL); } + @RequestMapping(value = "/migration", method = RequestMethod.POST) + @ResponseBody + public void clearCacheForCubeMigration(@RequestBody CubeMigrationRequest request) throws IOException { + cacheService.clearCacheForCubeMigration(request.getCube(), request.getProject(), request.getModel(), request.getTableToProjects()); + + cacheService.cleanDataCache(request.getProject()); + } + public void setCacheService(CacheService cacheService) { this.cacheService = cacheService; } diff --git a/server-base/src/main/java/org/apache/kylin/rest/request/CubeMigrationRequest.java b/server-base/src/main/java/org/apache/kylin/rest/request/CubeMigrationRequest.java new file mode 100644 index 000000000..175fb59a9 --- /dev/null +++ b/server-base/src/main/java/org/apache/kylin/rest/request/CubeMigrationRequest.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +package org.apache.kylin.rest.request; + +import java.util.Map; + +public class CubeMigrationRequest { + private String cube; + private String model; + private String project; + + private Map tableToProjects;//For KYLIN-2717 compatibility, the project of old table will be NULL + + public String getProject() { + return project; + } + + public void setProject(String project) { + this.project = project; + } + + public String getCube() { + return cube; + } + + public void setCube(String cube) { + this.cube = cube; + } + + public String getModel() { + return model; + } + + public void setModel(String model) { + this.model = model; + } + + public Map getTableToProjects() { + return tableToProjects; + } + + public void setTableToProjects(Map tableToProjects) { + this.tableToProjects = tableToProjects; + } + +} diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/CacheService.java b/server-base/src/main/java/org/apache/kylin/rest/service/CacheService.java index 98e06e075..beb5a8e7e 100644 --- a/server-base/src/main/java/org/apache/kylin/rest/service/CacheService.java +++ b/server-base/src/main/java/org/apache/kylin/rest/service/CacheService.java @@ -20,6 +20,7 @@ package org.apache.kylin.rest.service; import java.io.IOException; +import java.util.Map; import org.apache.kylin.metadata.cachesync.Broadcaster; import org.apache.kylin.metadata.cachesync.Broadcaster.Event; import org.slf4j.Logger; @@ -112,7 +113,7 @@ public class CacheService extends BasicService implements InitializingBean { broadcaster.notifyListener(entity, event, cacheKey); } - protected void cleanDataCache(String project) { + public void cleanDataCache(String project) { if (cacheManager != null) { logger.info("cleaning cache for project " + project + " (currently remove all entries)"); cacheManager.getCache(QueryService.SUCCESS_QUERY_CACHE).removeAll(); @@ -131,4 +132,35 @@ public class CacheService extends BasicService implements InitializingBean { } } + public void clearCacheForCubeMigration(String cube, String project, String model, Map tableToProjects) throws IOException { + //the metadata reloading must be in order + + //table must before model + for (Map.Entry entry : tableToProjects.entrySet()) { + //For KYLIN-2717 compatibility, use tableProject not project + getTableManager().reloadSourceTable(entry.getKey(), entry.getValue()); + getTableManager().reloadTableExt(entry.getKey(), entry.getValue()); + } + logger.info("reload table cache done"); + + //ProjectInstance cache must before cube and model cache, as the new cubeDesc init and model reloading relays on latest ProjectInstance cache + getProjectManager().reloadProjectQuietly(project); + logger.info("reload project cache done"); + + //model must before cube desc + getDataModelManager().reloadDataModel(model); + logger.info("reload model cache done"); + + //cube desc must before cube instance + getCubeDescManager().reloadCubeDescLocal(cube); + logger.info("reload cubeDesc cache done"); + + getCubeManager().reloadCubeQuietly(cube); + logger.info("reload cube cache done"); + + //reload project l2cache again after cube cache, because the project L2 cache relay on latest cube cache + getProjectManager().reloadProjectL2Cache(project); + logger.info("reload project l2cache done"); + } + } diff --git a/tool/src/main/java/org/apache/kylin/tool/CubeMigrationCLI.java b/tool/src/main/java/org/apache/kylin/tool/CubeMigrationCLI.java index f95139ed3..66b8016b2 100644 --- a/tool/src/main/java/org/apache/kylin/tool/CubeMigrationCLI.java +++ b/tool/src/main/java/org/apache/kylin/tool/CubeMigrationCLI.java @@ -54,7 +54,6 @@ import org.apache.kylin.dict.lookup.SnapshotManager; import org.apache.kylin.dict.lookup.SnapshotTable; import org.apache.kylin.engine.mr.JobBuilderSupport; import org.apache.kylin.metadata.MetadataConstants; -import org.apache.kylin.metadata.cachesync.Broadcaster; import org.apache.kylin.metadata.model.DataModelDesc; import org.apache.kylin.metadata.model.IStorageAware; import org.apache.kylin.metadata.model.SegmentStatusEnum; @@ -117,7 +116,7 @@ public class CubeMigrationCLI extends AbstractApplication { + "realExecute: if false, just print the operations to take, if true, do the real migration. \n"); } - + public void moveCube(KylinConfig srcCfg, KylinConfig dstCfg, String cubeName, String projectName, String copyAcl, String purgeAndDisable, String overwriteIfExists, String realExecute) throws IOException, InterruptedException { @@ -153,12 +152,12 @@ public class CubeMigrationCLI extends AbstractApplication { if (Boolean.parseBoolean(realExecute) == true) { doOpts(); checkMigrationSuccess(dstConfig, cubeName, true); - updateMeta(dstConfig); + updateMeta(dstConfig, projectName, cubeName, cube.getModel()); } else { showOpts(); } } - + public void moveCube(String srcCfgUri, String dstCfgUri, String cubeName, String projectName, String copyAcl, String purgeAndDisable, String overwriteIfExists, String realExecute) throws IOException, InterruptedException { @@ -571,7 +570,7 @@ public class CubeMigrationCLI extends AbstractApplication { } } } - + private String renameTableWithinProject(String srcItem) { if (dstProject != null && srcItem.contains(ResourceStore.TABLE_RESOURCE_ROOT)) { String tableIdentity = TableDesc.parseResourcePath(srcItem).getFirst(); @@ -583,13 +582,18 @@ public class CubeMigrationCLI extends AbstractApplication { return srcItem; } - private void updateMeta(KylinConfig config) { + private void updateMeta(KylinConfig config, String projectName, String cubeName, DataModelDesc model) { String[] nodes = config.getRestServers(); + Map tableToProjects = new HashMap<>(); + for (TableRef tableRef : model.getAllTables()) { + tableToProjects.put(tableRef.getTableIdentity(), tableRef.getTableDesc().getProject()); + } + for (String node : nodes) { RestClient restClient = new RestClient(node); try { logger.info("update meta cache for " + node); - restClient.wipeCache(Broadcaster.SYNC_ALL, Broadcaster.Event.UPDATE.getType(), Broadcaster.SYNC_ALL); + restClient.clearCacheForCubeMigration(cubeName, projectName, model.getName(), tableToProjects); } catch (IOException e) { logger.error(e.getMessage()); } -- 2.14.3 (Apple Git-98)