From adaa7e188cdf5993458e74d207548caea53f7a60 Mon Sep 17 00:00:00 2001 From: Zhong Date: Mon, 23 Jul 2018 14:37:34 +0800 Subject: [PATCH] KYLIN-3463 use LazyOutputFormat to prevent to create zero-sized default output for optimize jobs --- .../apache/kylin/engine/mr/steps/FilterRecommendCuboidDataJob.java | 4 +++- .../org/apache/kylin/engine/mr/steps/UpdateOldCuboidShardJob.java | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FilterRecommendCuboidDataJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FilterRecommendCuboidDataJob.java index 97f9dc1..b56434b 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FilterRecommendCuboidDataJob.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FilterRecommendCuboidDataJob.java @@ -25,6 +25,7 @@ import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat; import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; +import org.apache.hadoop.mapreduce.lib.output.LazyOutputFormat; import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat; import org.apache.kylin.common.KylinConfig; import org.apache.kylin.cube.CubeInstance; @@ -78,7 +79,8 @@ public class FilterRecommendCuboidDataJob extends AbstractHadoopJob { job.setInputFormatClass(SequenceFileInputFormat.class); FileInputFormat.setInputPaths(job, input); // Output - job.setOutputFormatClass(SequenceFileOutputFormat.class); + //// prevent to create zero-sized default output + LazyOutputFormat.setOutputFormatClass(job, SequenceFileOutputFormat.class); FileOutputFormat.setOutputPath(job, output); // set job configuration diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateOldCuboidShardJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateOldCuboidShardJob.java index 0cd7264..b71e459 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateOldCuboidShardJob.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateOldCuboidShardJob.java @@ -25,6 +25,7 @@ import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat; import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; +import org.apache.hadoop.mapreduce.lib.output.LazyOutputFormat; import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat; import org.apache.kylin.common.KylinConfig; import org.apache.kylin.cube.CubeInstance; @@ -80,7 +81,8 @@ public class UpdateOldCuboidShardJob extends AbstractHadoopJob { job.setInputFormatClass(SequenceFileInputFormat.class); FileInputFormat.setInputPaths(job, input); // Output - job.setOutputFormatClass(SequenceFileOutputFormat.class); + //// prevent to create zero-sized default output + LazyOutputFormat.setOutputFormatClass(job, SequenceFileOutputFormat.class); FileOutputFormat.setOutputPath(job, output); // set job configuration -- 2.5.4 (Apple Git-61)