From 15c2c5ddb5a3136ec84c24c0dd4882f6e87eb811 Mon Sep 17 00:00:00 2001 From: kangkaisen Date: Sun, 18 Dec 2016 20:09:03 +0800 Subject: [PATCH] KYLIN-2304 Only copy latest version dict for global dict --- .../org/apache/kylin/dict/AppendTrieDictionary.java | 18 ++++++++++++------ .../main/java/org/apache/kylin/dict/CachedTreeMap.java | 2 +- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java b/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java index 32bfde6..8d39b0c 100644 --- a/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java +++ b/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java @@ -1206,13 +1206,14 @@ public class AppendTrieDictionary extends Dictionary { @Override public AppendTrieDictionary copyToAnotherMeta(KylinConfig srcConfig, KylinConfig dstConfig) throws IOException { + //copy appendDict Configuration conf = new Configuration(); - AppendTrieDictionary newDict = new AppendTrieDictionary(); - newDict.initParams(baseDir.replaceFirst(srcConfig.getHdfsWorkingDirectory(), dstConfig.getHdfsWorkingDirectory()), baseId, maxId, maxValueLength, nValues, bytesConverter); - newDict.initDictSliceMap((CachedTreeMap)dictSliceMap); - logger.info("Copy AppendDict from {} to {}", this.baseDir, newDict.baseDir); - Path srcPath = new Path(this.baseDir); - Path dstPath = new Path(newDict.baseDir); + Path base = new Path(baseDir); + FileSystem srcFs = FileSystem.get(base.toUri(), conf); + Path srcPath = CachedTreeMap.getLatestVersion(conf, srcFs, base); + Path dstPath = new Path(srcPath.toString().replaceFirst(srcConfig.getHdfsWorkingDirectory(), dstConfig.getHdfsWorkingDirectory())); + logger.info("Copy appendDict from {} to {}", srcPath, dstPath); + FileSystem dstFs = FileSystem.get(dstPath.toUri(), conf); if (dstFs.exists(dstPath)) { logger.info("Delete existing AppendDict {}", dstPath); @@ -1220,6 +1221,11 @@ public class AppendTrieDictionary extends Dictionary { } FileUtil.copy(FileSystem.get(srcPath.toUri(), conf), srcPath, FileSystem.get(dstPath.toUri(), conf), dstPath, false, true, conf); + // init new AppendTrieDictionary + AppendTrieDictionary newDict = new AppendTrieDictionary(); + newDict.initParams(baseDir.replaceFirst(srcConfig.getHdfsWorkingDirectory(), dstConfig.getHdfsWorkingDirectory()), baseId, maxId, maxValueLength, nValues, bytesConverter); + newDict.initDictSliceMap((CachedTreeMap) dictSliceMap); + return newDict; } diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/CachedTreeMap.java b/core-dictionary/src/main/java/org/apache/kylin/dict/CachedTreeMap.java index 6acf764..16b38b5 100644 --- a/core-dictionary/src/main/java/org/apache/kylin/dict/CachedTreeMap.java +++ b/core-dictionary/src/main/java/org/apache/kylin/dict/CachedTreeMap.java @@ -225,7 +225,7 @@ public class CachedTreeMap ext return getLatestVersion(conf, fs, baseDir).toUri().getPath(); } - private static Path getLatestVersion(Configuration conf, FileSystem fs, Path baseDir) throws IOException { + public static Path getLatestVersion(Configuration conf, FileSystem fs, Path baseDir) throws IOException { String[] versions = listAllVersions(fs, baseDir); if (versions.length > 0) { return new Path(versions[versions.length - 1]); -- 2.10.1 (Apple Git-78)