diff --git common/src/java/org/apache/hadoop/hive/conf/HiveConf.java common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 7a8517b..bae22fd 100644 --- common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -1535,7 +1535,7 @@ public void setSparkConfigUpdated(boolean isSparkConfigUpdated) { "compaction re-queued."), HIVE_COMPACTOR_CHECK_INTERVAL("hive.compactor.check.interval", "300s", - new TimeValidator(TimeUnit.SECONDS), + new TimeValidator(TimeUnit.SECONDS, 5L, true, 60*60*24*7L, true), "Time in seconds between checks to see if any tables or partitions need to be\n" + "compacted. This should be kept high because each check for compaction requires\n" + "many calls against the NameNode.\n" + diff --git common/src/java/org/apache/hadoop/hive/conf/Validator.java common/src/java/org/apache/hadoop/hive/conf/Validator.java index 04a305d..d567478 100644 --- common/src/java/org/apache/hadoop/hive/conf/Validator.java +++ common/src/java/org/apache/hadoop/hive/conf/Validator.java @@ -242,6 +242,13 @@ public TimeUnit getTimeUnit() { return timeUnit; } + public Long getMin(TimeUnit toTimeUnit) { + return min == null ? null : HiveConf.toTime(Long.toString(min), timeUnit, toTimeUnit); + } + public Long getMax(TimeUnit toTimeUnit) { + return max == null ? null : HiveConf.toTime(Long.toString(max), timeUnit, toTimeUnit); + } + @Override public String validate(String value) { try { @@ -258,6 +265,19 @@ public String validate(String value) { return null; } + /** + * returns a value adjust to fall withing valid range. + */ + public Long getValidValue(String value, TimeUnit toTimeUnit) { + long time = HiveConf.toTime(value, timeUnit, timeUnit); + if (min != null && (minInclusive ? time < min : time <= min)) { + return HiveConf.toTime(Long.toString(min), timeUnit, toTimeUnit); + } + if (max != null && (maxInclusive ? time > max : time >= max)) { + return HiveConf.toTime(Long.toString(max), timeUnit, toTimeUnit); + } + return HiveConf.toTime(value, timeUnit, toTimeUnit); + } public String toDescription() { String description = "Expects a time value with unit " + diff --git ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Initiator.java ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Initiator.java index 2d051fd..ca41aa7 100644 --- ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Initiator.java +++ ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Initiator.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hive.ql.txn.compactor; +import org.apache.hadoop.hive.conf.Validator; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.fs.FileStatus; @@ -165,7 +166,17 @@ public void run() { public void init(AtomicBoolean stop, AtomicBoolean looped) throws MetaException { super.init(stop, looped); checkInterval = - conf.getTimeVar(HiveConf.ConfVars.HIVE_COMPACTOR_CHECK_INTERVAL, TimeUnit.MILLISECONDS) ; + conf.getTimeVar(HiveConf.ConfVars.HIVE_COMPACTOR_CHECK_INTERVAL, TimeUnit.MILLISECONDS); + Validator.TimeValidator v = (Validator.TimeValidator) + HiveConf.ConfVars.HIVE_COMPACTOR_CHECK_INTERVAL.getValidator(); + String errorMsg = v.validate(checkInterval + "ms"); + if(errorMsg == null) { + return; + } + checkInterval = v.getValidValue(conf.getVar(HiveConf.ConfVars.HIVE_COMPACTOR_CHECK_INTERVAL), + TimeUnit.MILLISECONDS); + LOG.warn(HiveConf.ConfVars.HIVE_COMPACTOR_CHECK_INTERVAL.name() + " value is out of range: " + + errorMsg + " Will use: " + checkInterval + "ms"); } private void recoverFailedCompactions(boolean remoteOnly) throws MetaException { diff --git ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/TestInitiator.java ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/TestInitiator.java index e9b4154..b79f88f 100644 --- ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/TestInitiator.java +++ ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/TestInitiator.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hive.ql.txn.compactor; +import org.apache.hadoop.hive.conf.Validator; import org.junit.Assert; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -50,6 +51,13 @@ public void nothing() throws Exception { // survival test. startInitiator(); } + @Test + public void testConfig() { + conf.set(HiveConf.ConfVars.HIVE_COMPACTOR_CHECK_INTERVAL.name(), "1L"); + Validator.TimeValidator v = (Validator.TimeValidator)HiveConf.ConfVars.HIVE_COMPACTOR_CHECK_INTERVAL.getValidator(); + Assert.assertEquals("Expected in-range value", v.getMin(TimeUnit.MILLISECONDS), + v.getValidValue(conf.get(HiveConf.ConfVars.HIVE_COMPACTOR_CHECK_INTERVAL.name()), TimeUnit.MILLISECONDS)); + } @Test public void recoverFailedLocalWorkers() throws Exception {