From 08fc355c915e8bbc3e0ee94ff843eb5b5e4ce746 Mon Sep 17 00:00:00 2001 From: kangkaisen Date: Tue, 5 Dec 2017 20:37:08 +0800 Subject: [PATCH] KYLIN-2999 One click migrate cube in web --- .../org/apache/kylin/common/KylinConfigBase.java | 25 +++++++++++++ .../kylin/rest/controller/CubeController.java | 7 ++++ .../org/apache/kylin/rest/service/CubeService.java | 42 ++++++++++++++++++++++ webapp/app/js/controllers/cubes.js | 33 +++++++++++++++-- webapp/app/js/services/cubes.js | 3 +- webapp/app/js/services/kylinProperties.js | 8 +++++ webapp/app/partials/cubes/cubes.html | 1 + 7 files changed, 116 insertions(+), 3 deletions(-) diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java index e1a10a87e..ffa2d6c90 100644 --- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java @@ -1395,4 +1395,29 @@ abstract public class KylinConfigBase implements Serializable { return getOptional("kylin.core.metrics.subject-query-rpc", "METRICS_QUERY_RPC") + "_" + getKylinMetricsSubjectSuffix(); } + + // ============================================================================ + // tool + // ============================================================================ + public boolean isAllowAutoMigrateCube() { + return Boolean.parseBoolean(getOptional("kylin.tool.auto-migrate-cube.enabled", "false")); + } + + public boolean isAutoMigrateCubeCopyAcl() { + return Boolean.parseBoolean(getOptional("kylin.tool.auto-migrate-cube.copy-acl", "true")); + } + + public boolean isAutoMigrateCubePurge() { + return Boolean.parseBoolean(getOptional("kylin.tool.auto-migrate-cube.purge-src-cube", "true")); + } + + public String getAutoMigrateCubeSrcConfig() { + return getOptional("kylin.tool.auto-migrate-cube.src-config", ""); + } + + public String getAutoMigrateCubeDestConfig() { + return getOptional("kylin.tool.auto-migrate-cube.dest-config", ""); + } + + } diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java index 0e9f4baa9..908c25347 100644 --- a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java +++ b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java @@ -1017,6 +1017,13 @@ public class CubeController extends BasicController { } } + @RequestMapping(value = "/{cube}/{project}/migrate", method = { RequestMethod.POST }) + @ResponseBody + public void migrateCube(@PathVariable String cube, @PathVariable String project) { + CubeInstance cubeInstance = cubeService.getCubeManager().getCube(cube); + cubeService.migrateCube(cubeInstance, project); + } + public void setCubeService(CubeService cubeService) { this.cubeService = cubeService; } diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java index b86ff1db3..c38450370 100644 --- a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java +++ b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java @@ -30,6 +30,7 @@ import java.util.Set; import org.apache.commons.lang.StringUtils; import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.persistence.RootPersistentEntity; +import org.apache.kylin.common.util.CliCommandExecutor; import org.apache.kylin.cube.CubeInstance; import org.apache.kylin.cube.CubeManager; import org.apache.kylin.cube.CubeSegment; @@ -41,6 +42,7 @@ import org.apache.kylin.cube.model.CubeDesc; import org.apache.kylin.engine.EngineFactory; import org.apache.kylin.engine.mr.CubingJob; import org.apache.kylin.engine.mr.common.CuboidRecommenderUtil; +import org.apache.kylin.job.common.PatternedLogger; import org.apache.kylin.job.exception.JobException; import org.apache.kylin.job.execution.DefaultChainedExecutable; import org.apache.kylin.job.execution.ExecutableState; @@ -57,6 +59,7 @@ import org.apache.kylin.metadata.realization.RealizationType; import org.apache.kylin.rest.constant.Constant; import org.apache.kylin.rest.exception.BadRequestException; import org.apache.kylin.rest.exception.ForbiddenException; +import org.apache.kylin.rest.exception.InternalErrorException; import org.apache.kylin.rest.msg.Message; import org.apache.kylin.rest.msg.MsgPicker; import org.apache.kylin.rest.request.MetricsRequest; @@ -848,4 +851,43 @@ public class CubeService extends BasicService implements InitializingBean { Map> rollingUpCountSourceMap) throws IOException { return CuboidRecommenderUtil.getRecommendCuboidList(cube, hitFrequencyMap, rollingUpCountSourceMap); } + + @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN + " or hasPermission(#cube, 'ADMINISTRATION') or hasPermission(#cube, 'MANAGEMENT')") + public void migrateCube(CubeInstance cube, String projectName) { + KylinConfig config = cube.getConfig(); + try { + if (!config.isAllowAutoMigrateCube()) { + throw new InternalErrorException("This cube couldn't one click migrate cube, Please contact your ADMIN"); + } + + for (CubeSegment segment : cube.getSegments()) { + if (segment.getStatus() != SegmentStatusEnum.READY) { + throw new InternalErrorException("At least one segment is not in READY state. Please check whether there are Running or Error jobs."); + } + } + + String srcCfgUri = config.getAutoMigrateCubeSrcConfig(); + String dstCfgUri = config.getAutoMigrateCubeDestConfig(); + + String stringBuilder = ("%s/bin/kylin.sh org.apache.kylin.tool.CubeMigrationCLI %s %s %s %s %s %s true true"); + String cmd = String.format(stringBuilder, KylinConfig.getKylinHome(), srcCfgUri, dstCfgUri, cube.getName(), + projectName, config.isAutoMigrateCubeCopyAcl(), config.isAutoMigrateCubePurge()); + + logger.info("cmd: " + cmd); + + CliCommandExecutor exec = new CliCommandExecutor(); + PatternedLogger patternedLogger = new PatternedLogger(logger); + + //Avoid high concurrent affecting prod query + synchronized (AdminService.class) { + try { + exec.execute(cmd, patternedLogger); + } catch (IOException e) { + throw new InternalErrorException("Migrating cube fails, Please contact your ADMIN"); + } + } + } catch (Throwable e) { + throw new InternalErrorException(e.getMessage()); + } + } } diff --git a/webapp/app/js/controllers/cubes.js b/webapp/app/js/controllers/cubes.js index dfa26a4c5..b80af635d 100644 --- a/webapp/app/js/controllers/cubes.js +++ b/webapp/app/js/controllers/cubes.js @@ -293,6 +293,35 @@ KylinApp.controller('CubesCtrl', function ($scope, $q, $routeParams, $location, }); }; + $scope.isAutoMigrateCubeEnabled = function(){ + return kylinConfig.isAutoMigrateCubeEnabled(); + }; + + $scope.migrateCube = function (cube) { + SweetAlert.swal({ + title: '', + text: "The cube will overwrite the same cube in prod env" + + "\nMigrating cube will elapse a couple of minutes." + + "\nPlease wait.", + type: '', + showCancelButton: true, + confirmButtonColor: '#DD6B55', + confirmButtonText: "Yes", + closeOnConfirm: true + }, function(isConfirm) { + if(isConfirm){ + loadingRequest.show(); + CubeService.autoMigrate({cubeId: cube.name, propName: $scope.projectModel.selectedProject}, {}, function (result) { + loadingRequest.hide(); + SweetAlert.swal('Success!', cube.name + ' migrate successfully!', 'success'); + },function(e){ + loadingRequest.hide(); + SweetAlert.swal('Migrate failed!', "Please contact your ADMIN.", 'error'); + }); + } + }); + }; + $scope.startJobSubmit = function (cube) { $scope.loadDetail(cube).then(function () { @@ -464,8 +493,6 @@ KylinApp.controller('CubesCtrl', function ($scope, $q, $routeParams, $location, }); }); } - - $scope.cubeEdit = function (cube) { $location.path("cubes/edit/" + cube.name); } @@ -494,6 +521,8 @@ KylinApp.controller('CubesCtrl', function ($scope, $q, $routeParams, $location, }); }) } + + }); diff --git a/webapp/app/js/services/cubes.js b/webapp/app/js/services/cubes.js index 150d9320b..30c2a3f1d 100644 --- a/webapp/app/js/services/cubes.js +++ b/webapp/app/js/services/cubes.js @@ -77,6 +77,7 @@ KylinApp.factory('CubeService', ['$resource', function ($resource, config) { } } }, - optimize: {method: 'PUT', params: {action: 'optimize'}, isArray: false} + optimize: {method: 'PUT', params: {action: 'optimize'}, isArray: false}, + autoMigrate: {method: 'POST', params: {action: 'migrate'}, isArray: false} }); }]); diff --git a/webapp/app/js/services/kylinProperties.js b/webapp/app/js/services/kylinProperties.js index a2af4ede7..9b4d2de46 100644 --- a/webapp/app/js/services/kylinProperties.js +++ b/webapp/app/js/services/kylinProperties.js @@ -115,5 +115,13 @@ KylinApp.service('kylinConfig', function (AdminService, $log) { return true; } + this.isAutoMigrateCubeEnabled = function(){ + var status = this.getProperty("kylin.tool.auto-migrate-cube.enabled").trim(); + if(status && status =='true'){ + return true; + } + return false; + } + }); diff --git a/webapp/app/partials/cubes/cubes.html b/webapp/app/partials/cubes/cubes.html index 3e7e961cd..d608f147b 100644 --- a/webapp/app/partials/cubes/cubes.html +++ b/webapp/app/partials/cubes/cubes.html @@ -99,6 +99,7 @@
  • Enable
  • Purge
  • Clone
    • +
  • Migrate