From 35c24aa5c6cd24a158e2c5c602684ba1cbf6f526 Mon Sep 17 00:00:00 2001 From: Guangxu Cheng Date: Mon, 11 Mar 2019 17:37:06 +0800 Subject: [PATCH] KYLIN-3866 Whether to set mapreduce.application.classpath is determined by the user --- .../apache/kylin/common/KylinConfigBase.java | 5 +++++ .../engine/mr/common/AbstractHadoopJob.java | 17 ++++++++++------- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java index 81979dc4f..7113da683 100644 --- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java @@ -1341,6 +1341,11 @@ public abstract class KylinConfigBase implements Serializable { return Integer.parseInt(getOptional("kylin.engine.mr.yarn-check-interval-seconds", "10")); } + + public boolean isUseLocalClasspathEnabled() { + return Boolean.parseBoolean(getOptional("kylin.engine.mr.use-local-classpath", TRUE)); + } + // ============================================================================ // ENGINE.SPARK // ============================================================================ diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java index ca92dfd5b..76286e3f4 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java @@ -214,15 +214,18 @@ public abstract class AbstractHadoopJob extends Configured implements Tool { String kylinKafkaDependency = System.getProperty("kylin.kafka.dependency"); Configuration jobConf = job.getConfiguration(); - String classpath = jobConf.get(MAP_REDUCE_CLASSPATH); - if (classpath == null || classpath.length() == 0) { - logger.info("Didn't find " + MAP_REDUCE_CLASSPATH + + if (kylinConf.isUseLocalClasspathEnabled()) { + String classpath = jobConf.get(MAP_REDUCE_CLASSPATH); + if (classpath == null || classpath.length() == 0) { + logger.info("Didn't find " + MAP_REDUCE_CLASSPATH + " in job configuration, will run 'mapred classpath' to get the default value."); - classpath = getDefaultMapRedClasspath(); - logger.info("The default mapred classpath is: " + classpath); - } + classpath = getDefaultMapRedClasspath(); + logger.info("The default mapred classpath is: " + classpath); + } - jobConf.set(MAP_REDUCE_CLASSPATH, classpath); + jobConf.set(MAP_REDUCE_CLASSPATH, classpath); + } logger.trace("Hadoop job classpath is: " + job.getConfiguration().get(MAP_REDUCE_CLASSPATH)); /* -- 2.17.1