diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index a8a87ad8c9c..60312b021f0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -2036,6 +2036,16 @@ public static boolean isAclEnabled(Configuration conf) { public static final float DEFAULT_NM_MAX_PER_DISK_UTILIZATION_PERCENTAGE = 90.0F; + /** + * If the maximum percentage of disk space that can be enabled. + */ + public static final String NM_PER_DISK_UTILIZATION_PERCENTAGE_ENABLED = + NM_DISK_HEALTH_CHECK_PREFIX + + "disk-utilization-percentage-enabled"; + + public static final + boolean DEFAULT_NM_PER_DISK_UTILIZATION_PERCENTAGE_ENABLED = true; + /** * The low threshold percentage of disk space used when an offline disk is * marked as online. Values can range from 0.0 to 100.0. The value shouldn't @@ -2054,6 +2064,17 @@ public static boolean isAclEnabled(Configuration conf) { */ public static final String NM_MIN_PER_DISK_FREE_SPACE_MB = NM_DISK_HEALTH_CHECK_PREFIX + "min-free-space-per-disk-mb"; + + /** + * If the minimum space of disk can be enabled. + */ + public static final String NM_PER_DISK_FREE_SPACE_ENABLED = + NM_DISK_HEALTH_CHECK_PREFIX + + "disk-free-space-enabled"; + + public static final boolean DEFAULT_NM_PER_DISK_FREE_SPACE_ENABLED = + true; + /** * The minimum space that must be available on an offline * disk for it to be marked as online. The value should not be less diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index f7d9fc1d2b0..236754209ba 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -1809,6 +1809,20 @@ 0.25 + + If the maximum percentage of disk space + utilization is enabled for disk health check. + yarn.nodemanager.disk-health-checker.disk-utilization-percentage-enabled + true + + + + The minimum space on a disk is enabled + for disk health check. + yarn.nodemanager.disk-health-checker.disk-free-space-enabled + true + + The maximum percentage of disk space utilization allowed after which a disk is marked as bad. Values can range from 0.0 to 100.0. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DirectoryCollection.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DirectoryCollection.java index 5b32e0e2d84..c7321c66670 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DirectoryCollection.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DirectoryCollection.java @@ -59,6 +59,9 @@ private final Configuration conf; private final DiskValidator diskValidator; + + private final boolean diskUtilizationPercentageEnabled; + private final boolean diskFreeSpaceEnabled; /** * The enum defines disk failure type. */ @@ -239,6 +242,17 @@ public DirectoryCollection(String[] dirs, throw new YarnRuntimeException(e); } + diskUtilizationPercentageEnabled = conf. + getBoolean(YarnConfiguration. + NM_PER_DISK_UTILIZATION_PERCENTAGE_ENABLED, + YarnConfiguration. + DEFAULT_NM_PER_DISK_UTILIZATION_PERCENTAGE_ENABLED); + diskFreeSpaceEnabled = conf. + getBoolean(YarnConfiguration. + NM_PER_DISK_FREE_SPACE_ENABLED, + YarnConfiguration. + DEFAULT_NM_PER_DISK_FREE_SPACE_ENABLED); + localDirs = new ArrayList<>(Arrays.asList(dirs)); errorDirs = new ArrayList<>(); fullDirs = new ArrayList<>(); @@ -520,7 +534,9 @@ boolean checkDirs() { diskUtilizationPercentageCutoffHigh : diskUtilizationPercentageCutoffLow; long diskFreeSpaceCutoff = goodDirs.contains(dir) ? diskFreeSpaceCutoffLow : diskFreeSpaceCutoffHigh; - if (isDiskUsageOverPercentageLimit(testDir, + + if (diskUtilizationPercentageEnabled + && isDiskUsageOverPercentageLimit(testDir, diskUtilizationPercentageCutoff)) { msg = "used space above threshold of " @@ -529,7 +545,8 @@ boolean checkDirs() { ret.put(dir, new DiskErrorInformation(DiskErrorCause.DISK_FULL, msg)); continue; - } else if (isDiskFreeSpaceUnderLimit(testDir, diskFreeSpaceCutoff)) { + } else if (diskFreeSpaceEnabled + && isDiskFreeSpaceUnderLimit(testDir, diskFreeSpaceCutoff)) { msg = "free space below limit of " + diskFreeSpaceCutoff + "MB"; @@ -613,6 +630,16 @@ long getDiskUtilizationSpaceCutoffHigh() { return diskFreeSpaceCutoffHigh; } + @VisibleForTesting + boolean getDiskUtilizationPercentageEnabled() { + return diskUtilizationPercentageEnabled; + } + + @VisibleForTesting + boolean getDiskFreeSpaceEnabled() { + return diskFreeSpaceEnabled; + } + public void setDiskUtilizationSpaceCutoff(long freeSpaceCutoff) { setDiskUtilizationSpaceCutoff(freeSpaceCutoff, freeSpaceCutoff);