diff --git hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/BasicDiskValidator.java hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/BasicDiskValidator.java new file mode 100644 index 0000000..0e5b18b --- /dev/null +++ hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/BasicDiskValidator.java @@ -0,0 +1,30 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.util; + +import org.apache.hadoop.util.DiskChecker.DiskErrorException; + +import java.io.File; + +public class BasicDiskValidator implements DiskValidator { + public static final String NAME = "basic"; + + public void checkStatus(File dir) throws DiskErrorException { + DiskChecker.checkDir(dir); + } +} diff --git hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DiskValidator.java hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DiskValidator.java new file mode 100644 index 0000000..f68f05e --- /dev/null +++ hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DiskValidator.java @@ -0,0 +1,41 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.util; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.util.DiskChecker.DiskErrorException; + +import java.io.File; + +/** + * A {@link DiskValidator} is the interface of a disk validator. + * + * The {@link #checkStatus(File)} operation checks status of a file/dir. + * + */ +@InterfaceAudience.Public +@InterfaceStability.Evolving +public interface DiskValidator { + /** + * Check the status of a file/dir + * @param dir a file/dir + * @throws DiskErrorException if any disk error + */ + void checkStatus(File dir) throws DiskErrorException; +} diff --git hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DiskValidatorFactory.java hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DiskValidatorFactory.java new file mode 100644 index 0000000..92a0e85 --- /dev/null +++ hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DiskValidatorFactory.java @@ -0,0 +1,72 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.util; + +import org.apache.hadoop.util.DiskChecker.DiskErrorException; + +import java.util.concurrent.ConcurrentHashMap; + +public class DiskValidatorFactory { + private static final + ConcurrentHashMap, DiskValidator> + instances = new ConcurrentHashMap<>(); + + /** + * Returns a {@link DiskValidator} instance corresponding to the passed clazz + */ + public static DiskValidator + getInstance(Class clazz) { + DiskValidator diskValidator = ReflectionUtils.newInstance(clazz, null); + DiskValidator diskValidatorRet = + instances.putIfAbsent(clazz, diskValidator); + if(diskValidatorRet != null) { + return diskValidatorRet; + } + return diskValidator; + } + + /** + * Returns {@link DiskValidator} instance corresponding to its name. + * DiskValidator can be "basic" for BasicDiskValidator; "read-write" for + * the ReadWriteDiskValidator. + * @param diskValidator canonical class name, or "basic" or "read-write". + * @throws DiskErrorException if if the class cannot be located + */ + @SuppressWarnings("unchecked") + public static DiskValidator getInstance(String diskValidator) + throws DiskErrorException { + @SuppressWarnings("rawtypes") + Class clazz; + String text = StringUtils.toLowerCase(diskValidator); + + if (text.equalsIgnoreCase(BasicDiskValidator.NAME)) { + clazz = BasicDiskValidator.class; + } else if (text.equalsIgnoreCase(ReadWriteDiskValidator.NAME)) { + clazz = ReadWriteDiskValidator.class; + } else { + try { + clazz = Class.forName(diskValidator); + } catch (ClassNotFoundException cnfe) { + throw new DiskErrorException(diskValidator + + " DiskValidator class not found!"); + } + } + + return getInstance(clazz); + } +} diff --git hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ReadWriteDiskValidator.java hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ReadWriteDiskValidator.java new file mode 100644 index 0000000..04701c3 --- /dev/null +++ hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ReadWriteDiskValidator.java @@ -0,0 +1,29 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.util; + +import org.apache.hadoop.util.DiskChecker.DiskErrorException; + +import java.io.File; + +public class ReadWriteDiskValidator implements DiskValidator { + public static final String NAME = "read-write"; + + public void checkStatus(File dir) throws DiskErrorException{ + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 17fbbc3..95a171b 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -2584,6 +2584,10 @@ public static boolean areNodeLabelsEnabled( public static final String TIMELINE_XFS_OPTIONS = TIMELINE_XFS_PREFIX + "xframe-options"; + // Disk Validator + public static final String DISK_VALIDATOR = "disk-validator"; + public static final String DEFAULT_DISK_VALIDATOR = "basic"; + public YarnConfiguration() { super(); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 703c03d..a3e8a38 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -2803,4 +2803,12 @@ yarn.resourcemanager.node-removal-untracked.timeout-ms 60000 + + + + The name of disk validator. + + disk-validator + basic + diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java index 927699e..f18be20 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java @@ -50,7 +50,7 @@ import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.security.token.TokenIdentifier; -import org.apache.hadoop.util.DiskChecker; +import org.apache.hadoop.util.DiskValidatorFactory; import org.apache.hadoop.util.concurrent.HadoopExecutors; import org.apache.hadoop.yarn.YarnUncaughtExceptionHandler; import org.apache.hadoop.yarn.api.records.LocalResource; @@ -197,7 +197,9 @@ ExecutorService createDownloadThreadPool() { Callable download(Path path, LocalResource rsrc, UserGroupInformation ugi) throws IOException { - DiskChecker.checkDir(new File(path.toUri().getRawPath())); + DiskValidatorFactory.getInstance( + conf.get(YarnConfiguration.DISK_VALIDATOR)). + checkStatus(new File(path.toUri().getRawPath())); return new FSDownload(lfs, ugi, conf, path, rsrc); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java index b2413ad..3701c37 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java @@ -71,7 +71,7 @@ import org.apache.hadoop.security.token.TokenIdentifier; import org.apache.hadoop.service.AbstractService; import org.apache.hadoop.service.CompositeService; -import org.apache.hadoop.util.DiskChecker; +import org.apache.hadoop.util.DiskValidatorFactory; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.util.concurrent.HadoopExecutors; import org.apache.hadoop.util.concurrent.HadoopScheduledThreadPoolExecutor; @@ -833,7 +833,9 @@ public void addResource(LocalizerResourceRequestEvent request) { publicRsrc.getPathForLocalization(key, publicRootPath, delService); if (!publicDirDestPath.getParent().equals(publicRootPath)) { - DiskChecker.checkDir(new File(publicDirDestPath.toUri().getPath())); + DiskValidatorFactory.getInstance( + conf.get(YarnConfiguration.DISK_VALIDATOR)).checkStatus( + new File(publicDirDestPath.toUri().getPath())); } // explicitly synchronize pending here to avoid future task