diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/SharedCacheClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/SharedCacheClient.java index 7cbe0e1..6e49672 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/SharedCacheClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/SharedCacheClient.java @@ -63,14 +63,22 @@ public SharedCacheClient(String name) { * exist, null is returned instead. *

* + *

+ * Additionally, a name for the resource should be specified. A fragment will + * be added to the path with the desired name. This ensures that if the + * returned path is used to create a LocalResource, then the symlink created + * during YARN localization will match the name specified. + *

+ * * @param applicationId ApplicationId of the application using the resource * @param resourceKey the key (i.e. checksum) that identifies the resource + * @param resourceName the desired name of the resource * @return Path to the resource, or null if it does not exist */ @Public @Unstable - public abstract Path use(ApplicationId applicationId, String resourceKey) - throws YarnException; + public abstract Path use(ApplicationId applicationId, String resourceKey, + String resourceName) throws YarnException; /** *

diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/SharedCacheClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/SharedCacheClientImpl.java index 0a61ee0..0426ed0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/SharedCacheClientImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/SharedCacheClientImpl.java @@ -21,6 +21,8 @@ import java.io.IOException; import java.net.InetSocketAddress; +import java.net.URI; +import java.net.URISyntaxException; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -111,8 +113,8 @@ protected void stopClientProxy() { } @Override - public Path use(ApplicationId applicationId, String resourceKey) - throws YarnException { + public Path use(ApplicationId applicationId, String resourceKey, + String resourceName) throws YarnException { Path resourcePath = null; UseSharedCacheResourceRequest request = Records.newRecord( UseSharedCacheResourceRequest.class); @@ -129,6 +131,24 @@ public Path use(ApplicationId applicationId, String resourceKey) // We don't handle different exceptions separately at this point. throw new YarnException(e); } + if (resourcePath != null) { + // We are using the shared cache, and a preferred name has been specified. + // We need to set the fragment portion of the URI to preserve the desired + // name. + URI pathURI = resourcePath.toUri(); + try { + // We assume that there is no existing fragment in the URI since the + // shared cache manager does not use fragments. + pathURI = + new URI(pathURI.getScheme(), pathURI.getSchemeSpecificPart(), + resourceName); + resourcePath = new Path(pathURI); + } catch (URISyntaxException e) { + throw new YarnException( + "Could not create a new URI due to syntax errors: " + + pathURI.toString(), e); + } + } return resourcePath; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestSharedCacheClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestSharedCacheClientImpl.java index 3985e54..94ff3f7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestSharedCacheClientImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestSharedCacheClientImpl.java @@ -26,6 +26,7 @@ import java.io.DataOutputStream; import java.io.FileNotFoundException; import java.io.IOException; +import java.net.URI; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -113,8 +114,22 @@ public void testUse() throws Exception { response.setPath(file.toString()); when(cProtocol.use(isA(UseSharedCacheResourceRequest.class))).thenReturn( response); - Path newPath = client.use(mock(ApplicationId.class), "key"); - assertEquals(file, newPath); + Path newPath = client.use(mock(ApplicationId.class), "key", null); + assertEquals("The paths are not equal!", file, newPath); + } + + @Test + public void testUseWithResourceName() throws Exception { + Path file = new Path("viewfs://test/path"); + URI useUri = new URI("viewfs://test/path#linkName"); + Path usePath = new Path(useUri); + UseSharedCacheResourceResponse response = + new UseSharedCacheResourceResponsePBImpl(); + response.setPath(file.toString()); + when(cProtocol.use(isA(UseSharedCacheResourceRequest.class))).thenReturn( + response); + Path newPath = client.use(mock(ApplicationId.class), "key", "linkName"); + assertEquals("The paths are not equal!", usePath, newPath); } @Test(expected = YarnException.class) @@ -122,7 +137,7 @@ public void testUseError() throws Exception { String message = "Mock IOExcepiton!"; when(cProtocol.use(isA(UseSharedCacheResourceRequest.class))).thenThrow( new IOException(message)); - client.use(mock(ApplicationId.class), "key"); + client.use(mock(ApplicationId.class), "key", null); } @Test