From 01215c58c81100a02607b8862b232b37cb67ff73 Mon Sep 17 00:00:00 2001 From: gaodayue Date: Thu, 28 Apr 2016 10:51:46 +0800 Subject: [PATCH] KYLIN-1657 add configuration kylin.job.mapreduce.min.reducer.number --- .../src/main/java/org/apache/kylin/common/KylinConfigBase.java | 4 ++++ .../src/main/java/org/apache/kylin/engine/mr/steps/CuboidJob.java | 6 +++--- .../main/java/org/apache/kylin/engine/mr/steps/InMemCuboidJob.java | 6 +++--- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java index 97245ce..3901b57 100644 --- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java @@ -289,6 +289,10 @@ abstract public class KylinConfigBase implements Serializable { return Double.parseDouble(getOptional("kylin.job.mapreduce.default.reduce.count.ratio", "1.0")); } + public int getHadoopJobMinReducerNumber() { + return Integer.parseInt(getOptional("kylin.job.mapreduce.min.reducer.number", "1")); + } + public int getHadoopJobMaxReducerNumber() { return Integer.parseInt(getOptional("kylin.job.mapreduce.max.reducer.number", "500")); } diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidJob.java index ce353f7..a445f71 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidJob.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CuboidJob.java @@ -203,9 +203,9 @@ public class CuboidJob extends AbstractHadoopJob { numReduceTasks = numReduceTasks * 4; } - // at least 1 reducer - numReduceTasks = Math.max(1, numReduceTasks); - // no more than 5000 reducer by default + // at least 1 reducer by default + numReduceTasks = Math.max(kylinConfig.getHadoopJobMinReducerNumber(), numReduceTasks); + // no more than 500 reducer by default numReduceTasks = Math.min(kylinConfig.getHadoopJobMaxReducerNumber(), numReduceTasks); jobConf.setInt(MAPRED_REDUCE_TASKS, numReduceTasks); diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidJob.java index e7bbdf1..258ec95 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidJob.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/InMemCuboidJob.java @@ -158,9 +158,9 @@ public class InMemCuboidJob extends AbstractHadoopJob { // number of reduce tasks int numReduceTasks = (int) Math.round(totalSizeInM / perReduceInputMB); - // at least 1 reducer - numReduceTasks = Math.max(1, numReduceTasks); - // no more than 5000 reducer by default + // at least 1 reducer by default + numReduceTasks = Math.max(kylinConfig.getHadoopJobMinReducerNumber(), numReduceTasks); + // no more than 500 reducer by default numReduceTasks = Math.min(kylinConfig.getHadoopJobMaxReducerNumber(), numReduceTasks); logger.info("Having total map input MB " + Math.round(totalSizeInM)); -- 2.7.4 (Apple Git-66)