From d3a357605ffa0ca040f1eb00265310c854c7a8cb Mon Sep 17 00:00:00 2001 From: lptong Date: Wed, 8 Nov 2017 12:11:20 +0800 Subject: [PATCH] https://issues.apache.org/jira/browse/KYLIN-3016 StorageCleanup job doesn't clean up all the legacy fiels in a in Read/Write seperation environment --- .../org/apache/kylin/rest/job/StorageCleanupJob.java | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/server-base/src/main/java/org/apache/kylin/rest/job/StorageCleanupJob.java b/server-base/src/main/java/org/apache/kylin/rest/job/StorageCleanupJob.java index 6d1d44cea8..d1cb1767e2 100755 --- a/server-base/src/main/java/org/apache/kylin/rest/job/StorageCleanupJob.java +++ b/server-base/src/main/java/org/apache/kylin/rest/job/StorageCleanupJob.java @@ -31,10 +31,12 @@ import org.apache.commons.cli.Option; import org.apache.commons.cli.OptionBuilder; import org.apache.commons.cli.Options; +import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.util.AbstractApplication; import org.apache.kylin.common.util.CliCommandExecutor; @@ -71,8 +73,8 @@ protected boolean delete = false; protected boolean force = false; - protected static ExecutableManager executableManager = ExecutableManager - .getInstance(KylinConfig.getInstanceFromEnv()); + protected static ExecutableManager executableManager = ExecutableManager.getInstance(KylinConfig + .getInstanceFromEnv()); protected void cleanUnusedHBaseTables() throws IOException { KylinConfig config = KylinConfig.getInstanceFromEnv(); @@ -105,15 +107,19 @@ protected void execute(OptionsHelper optionsHelper) throws Exception { delete = Boolean.parseBoolean(optionsHelper.getOptionValue(OPTION_DELETE)); force = Boolean.parseBoolean(optionsHelper.getOptionValue(OPTION_FORCE)); cleanUnusedIntermediateHiveTable(); - cleanUnusedHdfsFiles(); + KylinConfig config = KylinConfig.getInstanceFromEnv(); + if (StringUtils.isNotEmpty(config.getHBaseClusterFs())) { + cleanUnusedHdfsFiles(HBaseConfiguration.create()); + } + Configuration conf = HadoopUtil.getCurrentConfiguration(); + cleanUnusedHdfsFiles(conf); cleanUnusedHBaseTables(); } - private void cleanUnusedHdfsFiles() throws IOException { - Configuration conf = HadoopUtil.getCurrentConfiguration(); + private void cleanUnusedHdfsFiles(Configuration conf) throws IOException { + JobEngineConfig engineConfig = new JobEngineConfig(KylinConfig.getInstanceFromEnv()); CubeManager cubeMgr = CubeManager.getInstance(KylinConfig.getInstanceFromEnv()); - FileSystem fs = HadoopUtil.getWorkingFileSystem(conf); List allHdfsPathsNeedToBeDeleted = new ArrayList(); // GlobFilter filter = new