From aad8e5b4b95607572147603bcc42404fe98ef791 Mon Sep 17 00:00:00 2001 From: John Zhao Date: Tue, 2 Feb 2016 13:45:50 +0800 Subject: [PATCH] Add conf/kylin_job_conf.xml for HiveColumnCardinalityJob --- .../src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java | 2 +- .../kylin/source/hive/cardinality/HiveColumnCardinalityJob.java | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/core-job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java b/core-job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java index b305749..546c033 100644 --- a/core-job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java +++ b/core-job/src/main/java/org/apache/kylin/job/engine/JobEngineConfig.java @@ -51,7 +51,7 @@ public class JobEngineConfig { private String getHadoopJobConfFilePath(RealizationCapacity capaticy, boolean appendSuffix) throws IOException { String hadoopJobConfFile; - if (appendSuffix) { + if (capaticy != null && appendSuffix) { hadoopJobConfFile = (HADOOP_JOB_CONF_FILENAME + "_" + capaticy.toString().toLowerCase() + ".xml"); } else { hadoopJobConfFile = (HADOOP_JOB_CONF_FILENAME + ".xml"); diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/HiveColumnCardinalityJob.java b/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/HiveColumnCardinalityJob.java index 2b35755..51eb0ef 100644 --- a/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/HiveColumnCardinalityJob.java +++ b/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/HiveColumnCardinalityJob.java @@ -29,10 +29,13 @@ import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat; +import org.apache.kylin.common.KylinConfig; import org.apache.kylin.engine.mr.IMRInput.IMRTableInputFormat; import org.apache.kylin.engine.mr.MRUtil; import org.apache.kylin.engine.mr.common.AbstractHadoopJob; import org.apache.kylin.engine.mr.common.BatchConstants; +import org.apache.kylin.job.engine.JobEngineConfig; +import org.apache.kylin.metadata.model.DataModelDesc; /** * This hadoop job will scan all rows of the hive table and then calculate the cardinality on each column. @@ -65,6 +68,10 @@ public class HiveColumnCardinalityJob extends AbstractHadoopJob { String jobName = JOB_TITLE + getOptionsAsString(); logger.info("Starting: " + jobName); Configuration conf = getConf(); + + JobEngineConfig jobEngineConfig = new JobEngineConfig(KylinConfig.getInstanceFromEnv()); + conf.addResource(jobEngineConfig.getHadoopJobConfFilePath(null)); + job = Job.getInstance(conf, jobName); setJobClasspath(job); -- 2.5.4 (Apple Git-61)