From e2089c2366e4f87bb941acf7a902153ad2fde4ae Mon Sep 17 00:00:00 2001 From: etherge Date: Tue, 30 Jan 2018 14:58:06 +0800 Subject: [PATCH] KYLIN-2999 fix cube auto-migration --- .../org/apache/kylin/common/KylinConfigBase.java | 26 ++++++----- .../org/apache/kylin/rest/service/CubeService.java | 51 ++++++++++++---------- webapp/app/js/controllers/cubes.js | 2 +- 3 files changed, 42 insertions(+), 37 deletions(-) diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java index 7b48935..eee2d09 100644 --- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java @@ -742,7 +742,7 @@ abstract public class KylinConfigBase implements Serializable { public String getHiveBeelineShell() { return getOptional("kylin.source.hive.beeline-shell", "beeline"); } - + public String getHiveBeelineParams() { return getOptional("kylin.source.hive.beeline-params", ""); } @@ -750,11 +750,11 @@ abstract public class KylinConfigBase implements Serializable { public boolean getEnableSparkSqlForTableOps() { return Boolean.parseBoolean(getOptional("kylin.source.hive.enable-sparksql-for-table-ops", "false")); } - + public String getSparkSqlBeelineShell() { return getOptional("kylin.source.hive.sparksql-beeline-shell", ""); } - + public String getSparkSqlBeelineParams() { return getOptional("kylin.source.hive.sparksql-beeline-params", ""); } @@ -1504,14 +1504,17 @@ abstract public class KylinConfigBase implements Serializable { } public String getPropertiesWhiteList() { - return getOptional("kylin.web.properties.whitelist", - "kylin.web.timezone,kylin.query.cache-enabled,kylin.env,kylin.web.hive-limit,kylin.storage.default,kylin.engine.default,kylin.web.link-hadoop,kylin.web.link-diagnostic," - + "kylin.web.contact-mail,kylin.web.help.length,kylin.web.help.0,kylin.web.help.1,kylin.web.help.2,kylin.web.help.3," - + "kylin.web.help,kylin.web.hide-measures,kylin.web.link-streaming-guide,kylin.server.external-acl-provider,kylin.security.profile," - + "kylin.htrace.show-gui-trace-toggle,kylin.web.export-allow-admin,kylin.web.export-allow-other," - + "kylin.cube.cubeplanner.enabled,kylin.web.dashboard-enabled"); + return getOptional("kylin.web.properties.whitelist", "kylin.web.timezone,kylin.query.cache-enabled,kylin.env," + + "kylin.web.hive-limit,kylin.storage.default," + + "kylin.engine.default,kylin.web.link-hadoop,kylin.web.link-diagnostic," + + "kylin.web.contact-mail,kylin.web.help.length,kylin.web.help.0,kylin.web.help.1,kylin.web.help.2," + + "kylin.web.help.3," + + "kylin.web.help,kylin.web.hide-measures,kylin.web.link-streaming-guide,kylin.server.external-acl-provider," + + "kylin.security.profile," + + "kylin.htrace.show-gui-trace-toggle,kylin.web.export-allow-admin,kylin.web.export-allow-other," + + "kylin.cube.cubeplanner.enabled,kylin.web.dashboard-enabled,kylin.tool.auto-migrate-cube.enabled"); } - + // ============================================================================ // RESTCLIENT // ============================================================================ @@ -1527,7 +1530,7 @@ abstract public class KylinConfigBase implements Serializable { // ============================================================================ // Metrics // ============================================================================ - + public String getCoadhaleMetricsReportClassesNames() { return getOptional("kylin.metrics.reporter-classes", "org.apache.kylin.common.metrics.metrics2.JsonFileMetricsReporter,org.apache.kylin.common.metrics.metrics2.JmxMetricsReporter"); @@ -1629,5 +1632,4 @@ abstract public class KylinConfigBase implements Serializable { return getOptional("kylin.tool.auto-migrate-cube.dest-config", ""); } - } diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java index c01308d..f1bff41 100644 --- a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java +++ b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java @@ -83,6 +83,7 @@ import org.springframework.security.access.prepost.PreAuthorize; import org.springframework.security.core.context.SecurityContextHolder; import org.springframework.stereotype.Component; +import com.google.common.base.Preconditions; import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; import com.google.common.collect.Lists; @@ -836,39 +837,41 @@ public class CubeService extends BasicService implements InitializingBean { return CuboidRecommenderUtil.getRecommendCuboidList(cube, hitFrequencyMap, rollingUpCountSourceMap); } - @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN + " or hasPermission(#cube, 'ADMINISTRATION') or hasPermission(#cube, 'MANAGEMENT')") + @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN + + " or hasPermission(#cube, 'ADMINISTRATION') or hasPermission(#cube, 'MANAGEMENT')") public void migrateCube(CubeInstance cube, String projectName) { KylinConfig config = cube.getConfig(); - try { - if (!config.isAllowAutoMigrateCube()) { - throw new InternalErrorException("This cube couldn't one click migrate cube, Please contact your ADMIN"); - } + if (!config.isAllowAutoMigrateCube()) { + throw new InternalErrorException("One click migration is disabled, please contact your ADMIN"); + } - for (CubeSegment segment : cube.getSegments()) { - if (segment.getStatus() != SegmentStatusEnum.READY) { - throw new InternalErrorException("At least one segment is not in READY state. Please check whether there are Running or Error jobs."); - } + for (CubeSegment segment : cube.getSegments()) { + if (segment.getStatus() != SegmentStatusEnum.READY) { + throw new InternalErrorException( + "At least one segment is not in READY state. Please check whether there are Running or Error jobs."); } + } - String srcCfgUri = config.getAutoMigrateCubeSrcConfig(); - String dstCfgUri = config.getAutoMigrateCubeDestConfig(); + String srcCfgUri = config.getAutoMigrateCubeSrcConfig(); + String dstCfgUri = config.getAutoMigrateCubeDestConfig(); - String stringBuilder = ("%s/bin/kylin.sh org.apache.kylin.tool.CubeMigrationCLI %s %s %s %s %s %s true true"); - String cmd = String.format(stringBuilder, KylinConfig.getKylinHome(), srcCfgUri, dstCfgUri, cube.getName(), - projectName, config.isAutoMigrateCubeCopyAcl(), config.isAutoMigrateCubePurge()); + Preconditions.checkArgument(StringUtils.isNotEmpty(srcCfgUri), "Source configuration should not be empty."); + Preconditions.checkArgument(StringUtils.isNotEmpty(dstCfgUri), + "Destination configuration should not be empty."); - logger.info("cmd: " + cmd); + String stringBuilder = ("%s/bin/kylin.sh org.apache.kylin.tool.CubeMigrationCLI %s %s %s %s %s %s true true"); + String cmd = String.format(stringBuilder, KylinConfig.getKylinHome(), srcCfgUri, dstCfgUri, cube.getName(), + projectName, config.isAutoMigrateCubeCopyAcl(), config.isAutoMigrateCubePurge()); - CliCommandExecutor exec = new CliCommandExecutor(); - PatternedLogger patternedLogger = new PatternedLogger(logger); + logger.info("One click migration cmd: " + cmd); - try { - exec.execute(cmd, patternedLogger); - } catch (IOException e) { - throw new InternalErrorException("Migrating cube fails, Please contact your ADMIN"); - } - } catch (Throwable e) { - throw new InternalErrorException(e.getMessage()); + CliCommandExecutor exec = new CliCommandExecutor(); + PatternedLogger patternedLogger = new PatternedLogger(logger); + + try { + exec.execute(cmd, patternedLogger); + } catch (IOException e) { + throw new InternalErrorException("Failed to perform one-click migrating", e); } } } diff --git a/webapp/app/js/controllers/cubes.js b/webapp/app/js/controllers/cubes.js index 7094ff8..a454e0a 100644 --- a/webapp/app/js/controllers/cubes.js +++ b/webapp/app/js/controllers/cubes.js @@ -18,7 +18,7 @@ 'use strict'; -KylinApp.controller('CubesCtrl', function ($scope, $q, $routeParams, $location, $modal, MessageService, CubeDescService, CubeService, JobService, UserService, ProjectService, SweetAlert, loadingRequest, $log, cubeConfig, ProjectModel, ModelService, MetaModel, CubeList,modelsManager,TableService) { +KylinApp.controller('CubesCtrl', function ($scope, $q, $routeParams, $location, $modal, MessageService, CubeDescService, CubeService, JobService, UserService, ProjectService, SweetAlert, loadingRequest, $log, cubeConfig, ProjectModel, ModelService, MetaModel, CubeList,modelsManager,TableService, kylinConfig) { $scope.cubeConfig = cubeConfig; $scope.cubeList = CubeList; -- 2.9.3 (Apple Git-75)