From 302ea24d1d39f40712cfcbc80840bf1098c1a307 Mon Sep 17 00:00:00 2001 From: kangkaisen Date: Mon, 4 Sep 2017 10:29:00 +0800 Subject: [PATCH] KYLIN-2993 Add special mr config for base cuboid step --- .../src/main/java/org/apache/kylin/common/KylinConfigBase.java | 4 ++++ .../kylin/storage/hbase/steps/HBaseMROutput2Transition.java | 9 +++++++++ 2 files changed, 13 insertions(+) diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java index e1a10a87e..8a8b3ffc6 100644 --- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java @@ -991,6 +991,10 @@ abstract public class KylinConfigBase implements Serializable { return getPropertiesByPrefix("kylin.engine.mr.uhc-config-override."); } + public Map getBaseCuboidMRConfigOverride() { + return getPropertiesByPrefix("kylin.engine.mr.base-cuboid-config-override."); + } + public Map getSparkConfigOverride() { return getPropertiesByPrefix("kylin.engine.spark-conf."); } diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMROutput2Transition.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMROutput2Transition.java index db3f7f4e4..8817cb2af 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMROutput2Transition.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMROutput2Transition.java @@ -19,6 +19,7 @@ package org.apache.kylin.storage.hbase.steps; import java.util.List; +import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -99,6 +100,14 @@ public class HBaseMROutput2Transition implements IMROutput2 { int level) throws Exception { int reducerNum = 1; Class mapperClass = job.getMapperClass(); + + //allow user specially set config for base cuboid step + if (mapperClass == HiveToBaseCuboidMapper.class) { + for (Map.Entry entry : segment.getConfig().getBaseCuboidMRConfigOverride().entrySet()) { + job.getConfiguration().set(entry.getKey(), entry.getValue()); + } + } + if (mapperClass == HiveToBaseCuboidMapper.class || mapperClass == NDCuboidMapper.class) { reducerNum = MapReduceUtil.getLayeredCubingReduceTaskNum(segment, cuboidScheduler, AbstractHadoopJob.getTotalMapInputMB(job), level); -- 2.13.6 (Apple Git-96)