From 191ec1a87bec5457d49b9ae0b1bde6327fd1eb6f Mon Sep 17 00:00:00 2001 From: lidongsjtu Date: Tue, 1 Dec 2015 16:32:17 +0800 Subject: [PATCH] KYLIN-1190 Make memory budget per query configurable --- common/src/main/java/org/apache/kylin/common/KylinConfig.java | 4 ++++ .../java/org/apache/kylin/storage/hbase/CubeStorageEngine.java | 10 +++++++--- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/common/src/main/java/org/apache/kylin/common/KylinConfig.java b/common/src/main/java/org/apache/kylin/common/KylinConfig.java index a8f7c5d..9645b81 100644 --- a/common/src/main/java/org/apache/kylin/common/KylinConfig.java +++ b/common/src/main/java/org/apache/kylin/common/KylinConfig.java @@ -492,6 +492,10 @@ public class KylinConfig { return Boolean.parseBoolean(this.getOptional("kylin.query.cache.enabled", "true")); } + public long getQueryMemBudget() { + return Long.parseLong(this.getOptional("kylin.query.mem.budget", String.valueOf(3L * 1024 * 1024 * 1024))); + } + public int getHBaseKeyValueSize() { return Integer.parseInt(this.getOptional("kylin.hbase.client.keyvalue.maxsize", "10485760")); } diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/CubeStorageEngine.java b/storage/src/main/java/org/apache/kylin/storage/hbase/CubeStorageEngine.java index ed12781..bbf088e 100644 --- a/storage/src/main/java/org/apache/kylin/storage/hbase/CubeStorageEngine.java +++ b/storage/src/main/java/org/apache/kylin/storage/hbase/CubeStorageEngine.java @@ -76,7 +76,6 @@ public class CubeStorageEngine implements IStorageEngine { private static final Logger logger = LoggerFactory.getLogger(CubeStorageEngine.class); private static final int MERGE_KEYRANGE_THRESHOLD = 100; - private static final long MEM_BUDGET_PER_QUERY = 3L * 1024 * 1024 * 1024; // 3G private final CubeInstance cubeInstance; private final CubeDesc cubeDesc; @@ -643,8 +642,13 @@ public class CubeStorageEngine implements IStorageEngine { } } - long rowEst = MEM_BUDGET_PER_QUERY / rowSizeEst; - context.setThreshold((int) rowEst); + long rowEst = this.cubeInstance.getConfig().getQueryMemBudget() / rowSizeEst; + if (rowEst > 0) { + logger.info("Memory budget is set to: " + rowEst); + context.setThreshold((int) rowEst); + } else { + logger.info("Memory budget is not set."); + } } private void setLimit(TupleFilter filter, StorageContext context) { -- 2.3.8 (Apple Git-58)