From 3f6a4b0eae3c25a38d25a3292abfec04bbdc1242 Mon Sep 17 00:00:00 2001 From: kangkaisen Date: Sun, 11 Dec 2016 20:49:28 +0800 Subject: [PATCH] KYLIN-2266 Reduce memory usage for building global dict --- .../src/main/java/org/apache/kylin/common/KylinConfigBase.java | 9 --------- .../main/java/org/apache/kylin/dict/AppendTrieDictionary.java | 6 ++---- .../java/org/apache/kylin/dict/AppendTrieDictionaryTest.java | 1 - 3 files changed, 2 insertions(+), 14 deletions(-) diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java index f46c185..d4272f9 100644 --- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java @@ -245,15 +245,6 @@ abstract public class KylinConfigBase implements Serializable { setProperty("kylin.dictionary.append-entry-size", String.valueOf(entrySize)); } - public int getAppendDictCacheSize() { - return Integer.parseInt(getOptional("kylin.dictionary.append-cache-size", "20")); - } - - // for test - public void setAppendDictCacheSize(int cacheSize) { - setProperty("kylin.dictionary.append-cache-size", String.valueOf(cacheSize)); - } - public int getCachedSnapshotMaxEntrySize() { return Integer.parseInt(getOptional("kylin.snapshot.max-cache-entry", "500")); } diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java b/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java index 35aa9a5..32bfde6 100644 --- a/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java +++ b/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java @@ -115,10 +115,9 @@ public class AppendTrieDictionary extends Dictionary { } public void initDictSliceMap(CachedTreeMap dictMap) throws IOException { - int cacheSize = KylinConfig.getInstanceFromEnv().getAppendDictCacheSize(); int maxVersions = KylinConfig.getInstanceFromEnv().getAppendDictMaxVersions(); long versionTTL = KylinConfig.getInstanceFromEnv().getAppendDictVersionTTL(); - CachedTreeMap newDictSliceMap = CachedTreeMap.CachedTreeMapBuilder.newBuilder().maxSize(cacheSize).baseDir(baseDir) + CachedTreeMap newDictSliceMap = CachedTreeMap.CachedTreeMapBuilder.newBuilder().maxSize(1).baseDir(baseDir) .immutable(true).maxVersions(maxVersions).versionTTL(versionTTL).keyClazz(DictSliceKey.class).valueClazz(DictSlice.class).build(); newDictSliceMap.loadEntry(dictMap); this.dictSliceMap = newDictSliceMap; @@ -930,11 +929,10 @@ public class AppendTrieDictionary extends Dictionary { this.bytesConverter = bytesConverter; MAX_ENTRY_IN_SLICE = KylinConfig.getInstanceFromEnv().getAppendDictEntrySize(); - int cacheSize = KylinConfig.getInstanceFromEnv().getAppendDictCacheSize(); int maxVersions = KylinConfig.getInstanceFromEnv().getAppendDictMaxVersions(); long versionTTL = KylinConfig.getInstanceFromEnv().getAppendDictVersionTTL(); // create a new cached map with baseDir - mutableDictSliceMap = CachedTreeMap.CachedTreeMapBuilder.newBuilder().maxSize(cacheSize).baseDir(baseDir) + mutableDictSliceMap = CachedTreeMap.CachedTreeMapBuilder.newBuilder().maxSize(1).baseDir(baseDir) .maxVersions(maxVersions).versionTTL(versionTTL).keyClazz(DictSliceKey.class).valueClazz(DictNode.class).immutable(false).build(); if (dictMapBytes != null) { ((Writable) mutableDictSliceMap).readFields(new DataInputStream(new ByteArrayInputStream(dictMapBytes))); diff --git a/core-dictionary/src/test/java/org/apache/kylin/dict/AppendTrieDictionaryTest.java b/core-dictionary/src/test/java/org/apache/kylin/dict/AppendTrieDictionaryTest.java index a7e8152..0776599 100644 --- a/core-dictionary/src/test/java/org/apache/kylin/dict/AppendTrieDictionaryTest.java +++ b/core-dictionary/src/test/java/org/apache/kylin/dict/AppendTrieDictionaryTest.java @@ -63,7 +63,6 @@ public class AppendTrieDictionaryTest { System.setProperty(KylinConfig.KYLIN_CONF, "../examples/test_case_data/localmeta"); KylinConfig config = KylinConfig.getInstanceFromEnv(); config.setAppendDictEntrySize(50000); - config.setAppendDictCacheSize(3); config.setProperty("kylin.env.hdfs-working-dir", BASE_DIR); } -- 2.9.3 (Apple Git-75)