From e6bcf3242bfce602c3c28c143291c6c2ed075dc4 Mon Sep 17 00:00:00 2001 From: "Ma,Gang" Date: Mon, 13 Nov 2017 15:20:27 +0800 Subject: [PATCH] KYLIN-3020 org.apache.hadoop.util.ToolRunner is not threadsafe and misused in kylin HadoopShellExecutable --- .../java/org/apache/kylin/engine/mr/common/HadoopShellExecutable.java | 4 ++-- .../main/java/org/apache/kylin/storage/hbase/steps/BulkLoadJob.java | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopShellExecutable.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopShellExecutable.java index 10a6e8c..f6609c2 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopShellExecutable.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopShellExecutable.java @@ -22,8 +22,8 @@ import java.io.PrintWriter; import java.io.StringWriter; import java.lang.reflect.Constructor; -import org.apache.hadoop.util.ToolRunner; import org.apache.kylin.common.util.ClassUtil; +import org.apache.kylin.engine.mr.MRUtil; import org.apache.kylin.engine.mr.exception.HadoopShellException; import org.apache.kylin.job.exception.ExecuteException; import org.apache.kylin.job.execution.AbstractExecutable; @@ -60,7 +60,7 @@ public class HadoopShellExecutable extends AbstractExecutable { int result; StringBuilder log = new StringBuilder(); try { - result = ToolRunner.run(job, args); + result = MRUtil.runMRJob(job, args); } catch (Exception ex) { logger.error("error execute " + this.toString(), ex); StringWriter stringWriter = new StringWriter(); diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/BulkLoadJob.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/BulkLoadJob.java index d1caccb..f0b77aa 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/BulkLoadJob.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/BulkLoadJob.java @@ -25,6 +25,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FsShell; import org.apache.hadoop.hbase.mapreduce.LoadIncrementalHFiles; import org.apache.hadoop.util.ToolRunner; +import org.apache.kylin.engine.mr.MRUtil; import org.apache.kylin.engine.mr.common.AbstractHadoopJob; import org.apache.kylin.storage.hbase.HBaseConnection; import org.slf4j.Logger; @@ -74,7 +75,7 @@ public class BulkLoadJob extends AbstractHadoopJob { newArgs[1] = tableName; logger.debug("Start to run LoadIncrementalHFiles"); - int ret = ToolRunner.run(new LoadIncrementalHFiles(conf), newArgs); + int ret = MRUtil.runMRJob(new LoadIncrementalHFiles(conf), newArgs); logger.debug("End to run LoadIncrementalHFiles"); return ret; } -- 2.6.4