From 5b05882cc9ba1828a8530db5ebdb5daca6e51998 Mon Sep 17 00:00:00 2001 From: wangxianbin1987 Date: Thu, 25 Feb 2016 19:22:34 +0800 Subject: [PATCH] KYLIN-579 Unload Hive table from kylin Signed-off-by: wangxianbin1987 --- .../java/org/apache/kylin/cube/CubeManager.java | 2 + .../org/apache/kylin/metadata/MetadataManager.java | 30 ++++++++++ .../kylin/metadata/project/ProjectManager.java | 12 ++++ .../kylin/rest/controller/TableController.java | 59 ++++++++++++++++++- .../org/apache/kylin/rest/service/CubeService.java | 14 +++++ .../apache/kylin/rest/service/ModelService.java | 15 +++++ .../apache/kylin/rest/service/ProjectService.java | 20 +++++++ .../kylin/source/hive/HiveSourceTableLoader.java | 6 ++ webapp/app/js/controllers/sourceMeta.js | 66 ++++++++++++++++++++++ webapp/app/js/services/tables.js | 1 + webapp/app/partials/tables/source_table_tree.html | 6 +- webapp/app/partials/tables/table_unload.html | 33 +++++++++++ 12 files changed, 259 insertions(+), 5 deletions(-) create mode 100644 webapp/app/partials/tables/table_unload.html diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java index 84dd30a..4951ce6 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java @@ -46,7 +46,9 @@ import org.apache.kylin.metadata.MetadataManager; import org.apache.kylin.metadata.model.SegmentStatusEnum; import org.apache.kylin.metadata.model.TableDesc; import org.apache.kylin.metadata.model.TblColRef; +import org.apache.kylin.metadata.project.ProjectInstance; import org.apache.kylin.metadata.project.ProjectManager; +import org.apache.kylin.metadata.project.RealizationEntry; import org.apache.kylin.metadata.realization.IRealization; import org.apache.kylin.metadata.realization.IRealizationConstants; import org.apache.kylin.metadata.realization.IRealizationProvider; diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java b/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java index 80ee8b3..9f2a934 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java @@ -197,6 +197,12 @@ public class MetadataManager { srcTableMap.put(srcTable.getIdentity(), srcTable); } + public void removeSourceTable(String tableIdentity) throws IOException { + String path = TableDesc.concatResourcePath(tableIdentity); + getStore().deleteResource(path); + srcTableMap.remove(tableIdentity); + } + private void init(KylinConfig config) throws IOException { this.config = config; this.srcTableMap = new CaseInsensitiveStringCache(config, Broadcaster.TYPE.TABLE); @@ -336,6 +342,24 @@ public class MetadataManager { return new ArrayList<>(ret); } + public boolean isTableInModel(String tableName, String projectName) throws IOException { + for(DataModelDesc modelDesc : getModels(projectName)) { + if(modelDesc.getAllTables().contains(tableName.toUpperCase())) { + return true; + } + } + return false; + } + + public boolean isTableInAnyModel(String tableName) { + for(DataModelDesc modelDesc : getModels()) { + if(modelDesc.getAllTables().contains(tableName.toUpperCase())){ + return true; + } + } + return false; + } + private void reloadAllDataModel() throws IOException { ResourceStore store = getStore(); logger.debug("Reloading DataModel from folder " + store.getReadableResourcePath(ResourceStore.DATA_MODEL_DESC_RESOURCE_ROOT)); @@ -441,6 +465,12 @@ public class MetadataManager { srcTableExdMap.put(tableId, tableExdProperties); } + public void removeTableExd(String tableIdentity) throws IOException { + String path = TableDesc.concatExdResourcePath(tableIdentity); + getStore().deleteResource(path); + srcTableExdMap.remove(tableIdentity); + } + public String appendDBName(String table) { if (table.indexOf(".") > 0) diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectManager.java b/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectManager.java index 45bbb1b..f73239c 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectManager.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectManager.java @@ -286,6 +286,18 @@ public class ProjectManager { return projectInstance; } + public void removeTableDescFromProject(String tableIdentities, String projectName) throws IOException { + MetadataManager metaMgr = getMetadataManager(); + ProjectInstance projectInstance = getProject(projectName); + TableDesc table = metaMgr.getTableDesc(tableIdentities); + if (table == null) { + throw new IllegalStateException("Cannot find table '" + table + "' in metadata manager"); + } + + projectInstance.removeTable(table.getIdentity()); + updateProject(projectInstance); + } + public List findProjects(RealizationType type, String realizationName) { List result = Lists.newArrayList(); for (ProjectInstance prj : projectMap.values()) { diff --git a/server/src/main/java/org/apache/kylin/rest/controller/TableController.java b/server/src/main/java/org/apache/kylin/rest/controller/TableController.java index 39af7db..98e8d58 100644 --- a/server/src/main/java/org/apache/kylin/rest/controller/TableController.java +++ b/server/src/main/java/org/apache/kylin/rest/controller/TableController.java @@ -21,6 +21,7 @@ package org.apache.kylin.rest.controller; import java.io.IOException; import java.util.*; +import com.google.common.collect.Sets; import org.apache.commons.lang.StringUtils; import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.util.JsonUtil; @@ -33,6 +34,8 @@ import org.apache.kylin.rest.request.CardinalityRequest; import org.apache.kylin.rest.request.StreamingRequest; import org.apache.kylin.rest.response.TableDescResponse; import org.apache.kylin.rest.service.CubeService; +import org.apache.kylin.rest.service.ModelService; +import org.apache.kylin.rest.service.ProjectService; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -55,6 +58,10 @@ public class TableController extends BasicController { @Autowired private CubeService cubeMgmtService; + @Autowired + private ProjectService projectService; + @Autowired + private ModelService modelService; /** * Get available table list of the input database @@ -124,21 +131,67 @@ public class TableController extends BasicController { cubeMgmtService.syncTableToProject(loaded, project); Map result = new HashMap(); result.put("result.loaded", loaded); - result.put("result.unloaded", new String[] {}); + result.put("result.unloaded", new String[]{}); + return result; + } + + @RequestMapping(value = "/{tables}/{project}", method = { RequestMethod.DELETE }) + @ResponseBody + public Map unLoadHiveTables(@PathVariable String tables, @PathVariable String project) { + Set unLoadSuccess = Sets.newHashSet(); + Set unLoadFail = Sets.newHashSet(); + Map result = new HashMap(); + for (String tableName : tables.split(",")) { + if (unLoadHiveTable(tableName, project)) { + unLoadSuccess.add(tableName); + } else { + unLoadFail.add(tableName); + } + } + result.put("result.unload.success", (String[]) unLoadSuccess.toArray(new String[unLoadSuccess.size()])); + result.put("result.unload.fail", (String[]) unLoadFail.toArray(new String[unLoadFail.size()])); return result; } + /** + * table may referenced by several projects, and kylin only keep one copy of meta for each table, + * that's why we have two if statement here. + * @param tableName + * @param project + * @return + */ + private boolean unLoadHiveTable(String tableName, String project) { + boolean rtn= false; + try { + if (!modelService.isTableInModel(tableName, project)) { + cubeMgmtService.removeTableFromProject(tableName, project); + rtn = true; + } + } catch (IOException e) { + logger.error(e.getMessage(), e); + } + if(!projectService.isTableInAnyProject(tableName) && !modelService.isTableInAnyModel(tableName)) { + try { + cubeMgmtService.unLoadHiveTable(tableName); + rtn = true; + } catch (IOException e) { + logger.error(e.getMessage(), e); + rtn = false; + } + } + return rtn; + } @RequestMapping(value = "/addStreamingSrc", method = { RequestMethod.POST }) @ResponseBody public Map addStreamingTable(@RequestBody StreamingRequest request) throws IOException { Map result = new HashMap(); String project = request.getProject(); - TableDesc desc = JsonUtil.readValue(request.getTableData(),TableDesc.class); + TableDesc desc = JsonUtil.readValue(request.getTableData(), TableDesc.class); desc.setUuid(UUID.randomUUID().toString()); MetadataManager metaMgr = MetadataManager.getInstance(KylinConfig.getInstanceFromEnv()); metaMgr.saveSourceTable(desc); - cubeMgmtService.syncTableToProject(new String[]{desc.getName()},project); + cubeMgmtService.syncTableToProject(new String[]{desc.getName()}, project); result.put("success","true"); return result; } diff --git a/server/src/main/java/org/apache/kylin/rest/service/CubeService.java b/server/src/main/java/org/apache/kylin/rest/service/CubeService.java index 5d2776f..0c57d00 100644 --- a/server/src/main/java/org/apache/kylin/rest/service/CubeService.java +++ b/server/src/main/java/org/apache/kylin/rest/service/CubeService.java @@ -557,10 +557,24 @@ public class CubeService extends BasicService { } @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN) + public void unLoadHiveTable(String tableName) throws IOException { + String[] dbTableName = HadoopUtil.parseHiveTableName(tableName); + tableName = dbTableName[0] + "." + dbTableName[1]; + HiveSourceTableLoader.unLoadHiveTable(tableName.toUpperCase()); + } + + @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN) public void syncTableToProject(String[] tables, String project) throws IOException { getProjectManager().addTableDescToProject(tables, project); } + @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN) + public void removeTableFromProject(String tableName, String projectName) throws IOException { + String[] dbTableName = HadoopUtil.parseHiveTableName(tableName); + tableName = dbTableName[0] + "." + dbTableName[1]; + getProjectManager().removeTableDescFromProject(tableName, projectName); + } + @PreAuthorize(Constant.ACCESS_HAS_ROLE_MODELER + " or " + Constant.ACCESS_HAS_ROLE_ADMIN) public void calculateCardinalityIfNotPresent(String[] tables, String submitter) throws IOException { MetadataManager metaMgr = getMetadataManager(); diff --git a/server/src/main/java/org/apache/kylin/rest/service/ModelService.java b/server/src/main/java/org/apache/kylin/rest/service/ModelService.java index 9dae312..9d8ccfb 100644 --- a/server/src/main/java/org/apache/kylin/rest/service/ModelService.java +++ b/server/src/main/java/org/apache/kylin/rest/service/ModelService.java @@ -23,6 +23,7 @@ import java.util.ArrayList; import java.util.List; import org.apache.kylin.cube.model.CubeDesc; +import org.apache.kylin.engine.mr.HadoopUtil; import org.apache.kylin.invertedindex.model.IIDesc; import org.apache.kylin.metadata.model.DataModelDesc; import org.apache.kylin.metadata.project.ProjectInstance; @@ -128,4 +129,18 @@ public class ModelService extends BasicService { accessService.clean(desc, true); } + + @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN + " or hasPermission(#desc, 'ADMINISTRATION') or hasPermission(#desc, 'MANAGEMENT')") + public boolean isTableInAnyModel(String tableName) { + String[] dbTableName = HadoopUtil.parseHiveTableName(tableName); + tableName = dbTableName[0] + "." + dbTableName[1]; + return getMetadataManager().isTableInAnyModel(tableName); + } + + @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN + " or hasPermission(#desc, 'ADMINISTRATION') or hasPermission(#desc, 'MANAGEMENT')") + public boolean isTableInModel(String tableName, String projectName) throws IOException { + String[] dbTableName = HadoopUtil.parseHiveTableName(tableName); + tableName = dbTableName[0] + "." + dbTableName[1]; + return getMetadataManager().isTableInModel(tableName, projectName); + } } diff --git a/server/src/main/java/org/apache/kylin/rest/service/ProjectService.java b/server/src/main/java/org/apache/kylin/rest/service/ProjectService.java index be70534..ad5a982 100644 --- a/server/src/main/java/org/apache/kylin/rest/service/ProjectService.java +++ b/server/src/main/java/org/apache/kylin/rest/service/ProjectService.java @@ -23,6 +23,7 @@ import java.util.Collections; import java.util.List; import org.apache.kylin.metadata.project.ProjectInstance; +import org.apache.kylin.metadata.project.ProjectManager; import org.apache.kylin.rest.constant.Constant; import org.apache.kylin.rest.exception.InternalErrorException; import org.apache.kylin.rest.request.CreateProjectRequest; @@ -104,4 +105,23 @@ public class ProjectService extends BasicService { accessService.clean(project, true); } + public boolean isTableInAnyProject(String tableName) { + for(ProjectInstance projectInstance : ProjectManager.getInstance(getConfig()).listAllProjects()) { + if(projectInstance.containsTable(tableName.toUpperCase())) { + return true; + } + } + return false; + } + + public boolean isTableInProject(String tableName, String projectName) { + ProjectInstance projectInstance = ProjectManager.getInstance(getConfig()).getProject(projectName); + if(projectInstance != null) { + if(projectInstance.containsTable(tableName.toUpperCase())) { + return true; + } + } + return false; + } + } diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java index bc722b3..f2f2d2a 100644 --- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java +++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSourceTableLoader.java @@ -78,6 +78,12 @@ public class HiveSourceTableLoader { return loadedTables; } + public static void unLoadHiveTable(String hiveTable) throws IOException { + MetadataManager metaMgr = MetadataManager.getInstance(KylinConfig.getInstanceFromEnv()); + metaMgr.removeSourceTable(hiveTable); + metaMgr.removeTableExd(hiveTable); + } + private static List extractHiveTables(String database, Set tables, KylinConfig config) throws IOException { List loadedTables = Lists.newArrayList(); diff --git a/webapp/app/js/controllers/sourceMeta.js b/webapp/app/js/controllers/sourceMeta.js index 8d9421f..9320f24 100755 --- a/webapp/app/js/controllers/sourceMeta.js +++ b/webapp/app/js/controllers/sourceMeta.js @@ -100,6 +100,25 @@ KylinApp }); }; + $scope.openUnLoadModal = function () { + $modal.open({ + templateUrl: 'removeHiveTable.html', + controller: ModalInstanceCtrl, + backdrop : 'static', + resolve: { + tableNames: function () { + return $scope.tableNames; + }, + projectName: function () { + return $scope.projectModel.selectedProject; + }, + scope: function () { + return $scope; + } + } + }); + }; + var ModalInstanceCtrl = function ($scope, $location, $modalInstance, tableNames, MessageService, projectName, scope) { $scope.tableNames = ""; $scope.projectName = projectName; @@ -152,6 +171,53 @@ KylinApp loadingRequest.hide(); }) } + + $scope.remove = function () { + if ($scope.tableNames.trim() === "") { + SweetAlert.swal('', 'Please input table(s) you want to synchronize.', 'info'); + return; + } + + if (!$scope.projectName) { + SweetAlert.swal('', 'Please choose your project first!.', 'info'); + return; + } + + $scope.cancel(); + loadingRequest.show(); + TableService.unLoadHiveTable({tableName: $scope.tableNames, action: projectName}, {}, function (result) { + var removedTableInfo = ""; + angular.forEach(result['result.unload.success'], function (table) { + removedTableInfo += "\n" + table; + }) + var unRemovedTableInfo = ""; + angular.forEach(result['result.unload.fail'], function (table) { + unRemovedTableInfo += "\n" + table; + }) + + if (result['result.unload.fail'].length != 0 && result['result.unload.success'].length == 0) { + SweetAlert.swal('Failed!', 'Failed to synchronize following table(s): ' + unRemovedTableInfo, 'error'); + } + if (result['result.unload.success'].length != 0 && result['result.unload.fail'].length == 0) { + SweetAlert.swal('Success!', 'The following table(s) have been successfully synchronized: ' + removedTableInfo, 'success'); + } + if (result['result.unload.success'].length != 0 && result['result.unload.fail'].length != 0) { + SweetAlert.swal('Partial unloaded!', 'The following table(s) have been successfully synchronized: ' + removedTableInfo + "\n\n Failed to synchronize following table(s):" + unRemovedTableInfo, 'warning'); + } + loadingRequest.hide(); + scope.aceSrcTbLoaded(true); + + }, function (e) { + if (e.data && e.data.exception) { + var message = e.data.exception; + var msg = !!(message) ? message : 'Failed to take action.'; + SweetAlert.swal('Oops...', msg, 'error'); + } else { + SweetAlert.swal('Oops...', "Failed to take action.", 'error'); + } + loadingRequest.hide(); + }) + } }; diff --git a/webapp/app/js/services/tables.js b/webapp/app/js/services/tables.js index 3b5e9f4..ca7fc42 100755 --- a/webapp/app/js/services/tables.js +++ b/webapp/app/js/services/tables.js @@ -23,6 +23,7 @@ KylinApp.factory('TableService', ['$resource', function ($resource, config) { getExd: {method: 'GET', params: {action: 'exd-map'}, isArray: false}, reload: {method: 'PUT', params: {action: 'reload'}, isArray: false}, loadHiveTable: {method: 'POST', params: {}, isArray: false}, + unLoadHiveTable: {method: 'DELETE', params: {}, isArray: false}, addStreamingSrc: {method: 'POST', params: {action:'addStreamingSrc'}, isArray: false}, genCardinality: {method: 'PUT', params: {action: 'cardinality'}, isArray: false} }); diff --git a/webapp/app/partials/tables/source_table_tree.html b/webapp/app/partials/tables/source_table_tree.html index 767eb43..4eddc4f 100755 --- a/webapp/app/partials/tables/source_table_tree.html +++ b/webapp/app/partials/tables/source_table_tree.html @@ -25,8 +25,9 @@
- - + + +
@@ -47,3 +48,4 @@
+
diff --git a/webapp/app/partials/tables/table_unload.html b/webapp/app/partials/tables/table_unload.html new file mode 100644 index 0000000..a1fcf6f --- /dev/null +++ b/webapp/app/partials/tables/table_unload.html @@ -0,0 +1,33 @@ + + + -- 2.6.3.windows.1