From 387f0acc15d4b652e2d7d1cc992a0eb563c883cc Mon Sep 17 00:00:00 2001 From: kangkaisen Date: Sat, 7 Jan 2017 19:16:30 +0800 Subject: [PATCH] KYLIN-2357 Make ERROR_RECORD_LOG_THRESHOLD configurable --- .../src/main/java/org/apache/kylin/common/KylinConfigBase.java | 4 ++++ .../main/java/org/apache/kylin/engine/mr/common/BatchConstants.java | 1 - .../java/org/apache/kylin/engine/mr/steps/BaseCuboidMapperBase.java | 2 +- .../apache/kylin/engine/mr/steps/FactDistinctColumnsMapperBase.java | 2 +- 4 files changed, 6 insertions(+), 3 deletions(-) diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java index 88c4dbc..82d9314 100644 --- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java @@ -412,6 +412,10 @@ abstract public class KylinConfigBase implements Serializable { return Integer.parseInt(getOptional("kylin.job.scheduler.default", "0")); } + public Integer getErrorRecordThreshold() { + return Integer.parseInt(getOptional("kylin.job.error-record-threshold", "0")); + } + // ============================================================================ // SOURCE.HIVE // ============================================================================ diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BatchConstants.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BatchConstants.java index 078d80f..0281539 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BatchConstants.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BatchConstants.java @@ -80,7 +80,6 @@ public interface BatchConstants { */ String MAPREDUCE_COUNTER_GROUP_NAME = "Cube Builder"; int NORMAL_RECORD_LOG_THRESHOLD = 100000; - int ERROR_RECORD_LOG_THRESHOLD = 100; /** * dictionaries builder class diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/BaseCuboidMapperBase.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/BaseCuboidMapperBase.java index 7b719e0..b7fa756 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/BaseCuboidMapperBase.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/BaseCuboidMapperBase.java @@ -194,7 +194,7 @@ abstract public class BaseCuboidMapperBase extends KylinMapper BatchConstants.ERROR_RECORD_LOG_THRESHOLD) { + if (errorRecordCounter > cubeSegment.getConfig().getErrorRecordThreshold()) { if (ex instanceof IOException) throw (IOException) ex; else if (ex instanceof RuntimeException) diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapperBase.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapperBase.java index 74aebb0..34624e3 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapperBase.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapperBase.java @@ -105,7 +105,7 @@ abstract public class FactDistinctColumnsMapperBase extends Kyli ex.printStackTrace(System.err); errorRecordCounter++; - if (errorRecordCounter > BatchConstants.ERROR_RECORD_LOG_THRESHOLD) { + if (errorRecordCounter > cubeSeg.getConfig().getErrorRecordThreshold()) { if (ex instanceof IOException) throw (IOException) ex; else if (ex instanceof RuntimeException) -- 2.10.1 (Apple Git-78)