diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleHandler.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleHandler.java index 801724851d3..698821908e9 100644 --- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleHandler.java +++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleHandler.java @@ -173,6 +173,14 @@ public Path getLocalPathForWrite(String path, long size) throws IOException { return new Path(ABS_LOG_DIR.getAbsolutePath()); } + + @Override + public Iterable getAllLocalPathsForRead(String path) + throws IOException { + ArrayList paths = new ArrayList<>(); + paths.add(new Path(ABS_LOG_DIR.getAbsolutePath())); + return paths; + } } private static class MockShuffleHandler2 extends diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/AuxiliaryLocalPathHandler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/AuxiliaryLocalPathHandler.java index 50feecf4535..452edce0f0b 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/AuxiliaryLocalPathHandler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/AuxiliaryLocalPathHandler.java @@ -55,4 +55,12 @@ * @throws IOException if the path creations fails */ Path getLocalPathForWrite(String path, long size) throws IOException; + + /** + * Get all paths from the local FS for reading for a given Auxiliary Service. + * @param path the requested path + * @return the complete path list to the file on a local disk as an Iterable + * @throws IOException if the file read encounters a problem + */ + Iterable getAllLocalPathsForRead(String path) throws IOException; } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LocalDirsHandlerService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LocalDirsHandlerService.java index b2bb4e378fe..74380ad95f9 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LocalDirsHandlerService.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LocalDirsHandlerService.java @@ -612,6 +612,10 @@ public Path getLocalPathForRead(String pathStr) throws IOException { return getPathToRead(pathStr, getLocalDirsForRead()); } + public Iterable getAllLocalPathsForRead(String pathStr) throws IOException { + return localDirsAllocator.getAllLocalPathsToRead(pathStr, getConfig()); + } + public Path getLogPathForWrite(String pathStr, boolean checkWrite) throws IOException { return logDirsAllocator.getLocalPathForWrite(pathStr, diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java index 8a12c3c5105..a91dcf54258 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java @@ -1638,6 +1638,11 @@ public Path getLocalPathForWrite(String path, long size) throws IOException { return dirhandlerService.getLocalPathForWrite(path, size, false); } + + @Override + public Iterable getAllLocalPathsForRead(String path) throws IOException { + return dirhandlerService.getAllLocalPathsForRead(path); + } } @SuppressWarnings("unchecked")