diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceAM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceAM.java index e9478f0..555a9fe 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceAM.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceAM.java @@ -339,7 +339,7 @@ public void testRecordTokensForContainers() throws Exception { MockServiceAM am = new MockServiceAM(exampleApp, dockerCred); ByteBuffer amCredBuffer = am.recordTokensForContainers(); Credentials amCreds = - DockerClientConfigHandler.getCredentialsFromTokensByteBuffer( + DockerClientConfigHandler.getDockerCredentialsFromTokensByteBuffer( amCredBuffer); assertEquals(2, amCreds.numberOfTokens()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/DockerClientConfigHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/DockerClientConfigHandler.java index 5522cf4..da42e62 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/DockerClientConfigHandler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/DockerClientConfigHandler.java @@ -131,7 +131,7 @@ public static Credentials readCredentialsFromConfigFile(Path configFile, * @param tokens the Tokens from the ContainerLaunchContext. * @return the Credentials object populated from the Tokens. */ - public static Credentials getCredentialsFromTokensByteBuffer( + private static Credentials getCredentialsFromTokensByteBuffer( ByteBuffer tokens) throws IOException { Credentials credentials = new Credentials(); DataInputByteBuffer dibb = new DataInputByteBuffer(); @@ -148,36 +148,59 @@ public static Credentials getCredentialsFromTokensByteBuffer( } /** + * Convert the Token ByteBuffer to the appropriate Credentials object, + * extracting only the Docker credentials from the Token ByteBuffer. + * + * @param tokens the Tokens from the ContainerLaunchContext. + * @return the Credentials object populated from the Tokens. + */ + public static Credentials getDockerCredentialsFromTokensByteBuffer( + ByteBuffer tokens) throws IOException { + Credentials allCreds = getCredentialsFromTokensByteBuffer(tokens); + Credentials dockerCreds = new Credentials(); + for (Token token : allCreds.getAllTokens()) { + if (token.getKind().equals(DockerCredentialTokenIdentifier.KIND)) { + DockerCredentialTokenIdentifier ti = + (DockerCredentialTokenIdentifier) token.decodeIdentifier(); + Text tokenAlias = + new Text(ti.getRegistryUrl() + "-" + ti.getApplicationId()); + dockerCreds.addToken(tokenAlias, token); + } + } + return dockerCreds; + } + + /** * Extract the Docker related tokens from the Credentials and write the Docker * client configuration to the supplied File. * * @param outConfigFile the File to write the Docker client configuration to. * @param credentials the populated Credentials object. * @throws IOException if the write fails. + * @throws IllegalArgumentException if a non-Docker token is supplied. */ public static void writeDockerCredentialsToPath(File outConfigFile, - Credentials credentials) throws IOException { + Credentials credentials) throws IOException, IllegalArgumentException { ObjectMapper mapper = new ObjectMapper(); ObjectNode rootNode = mapper.createObjectNode(); ObjectNode registryUrlNode = mapper.createObjectNode(); - boolean foundDockerCred = false; if (credentials.numberOfTokens() > 0) { for (Token tk : credentials.getAllTokens()) { - if (tk.getKind().equals(DockerCredentialTokenIdentifier.KIND)) { - foundDockerCred = true; - DockerCredentialTokenIdentifier ti = - (DockerCredentialTokenIdentifier) tk.decodeIdentifier(); - ObjectNode registryCredNode = mapper.createObjectNode(); - registryUrlNode.put(ti.getRegistryUrl(), registryCredNode); - registryCredNode.put(CONFIG_AUTH_KEY, - new String(tk.getPassword(), Charset.forName("UTF-8"))); - if (LOG.isDebugEnabled()) { - LOG.debug("Prepared token for write: " + tk.toString()); - } + if (!tk.getKind().equals(DockerCredentialTokenIdentifier.KIND)) { + throw new IllegalArgumentException( + "Token is not a Docker client credential token: " + + tk.toString()); + } + DockerCredentialTokenIdentifier ti = + (DockerCredentialTokenIdentifier) tk.decodeIdentifier(); + ObjectNode registryCredNode = mapper.createObjectNode(); + registryUrlNode.put(ti.getRegistryUrl(), registryCredNode); + registryCredNode.put(CONFIG_AUTH_KEY, + new String(tk.getPassword(), Charset.forName("UTF-8"))); + if (LOG.isDebugEnabled()) { + LOG.debug("Prepared token for write: " + tk.toString()); } } - } - if (foundDockerCred) { rootNode.put(CONFIG_AUTHS_KEY, registryUrlNode); String json = mapper.writerWithDefaultPrettyPrinter().writeValueAsString(rootNode); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/security/TestDockerClientConfigHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/security/TestDockerClientConfigHandler.java index c4cbe45..07828ac 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/security/TestDockerClientConfigHandler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/security/TestDockerClientConfigHandler.java @@ -24,6 +24,7 @@ import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.security.token.TokenIdentifier; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.util.DockerClientConfigHandler; import org.junit.Before; import org.junit.Test; @@ -32,6 +33,7 @@ import java.io.File; import java.io.FileWriter; import java.nio.ByteBuffer; +import java.nio.charset.Charset; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -93,7 +95,8 @@ public void testGetCredentialsFromTokensByteBuffer() throws Exception { credentials.writeTokenStorageToStream(dob); ByteBuffer tokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength()); Credentials credentialsOut = - DockerClientConfigHandler.getCredentialsFromTokensByteBuffer(tokens); + DockerClientConfigHandler.getDockerCredentialsFromTokensByteBuffer( + tokens); assertEquals(credentials.numberOfTokens(), credentialsOut.numberOfTokens()); for (Token tkIn : credentials.getAllTokens()) { DockerCredentialTokenIdentifier ti = @@ -126,4 +129,17 @@ public void testWriteDockerCredentialsToPath() throws Exception { assertTrue(fileContents.contains("foobarbaz")); assertTrue(fileContents.contains("bazbarfoo")); } + + @Test(expected = IllegalArgumentException.class) + public void testExceptionWritingNonDockerToken() throws Exception { + Credentials nonDockerCreds = new Credentials(); + TokenIdentifier tokenId = new AMRMTokenIdentifier( + ApplicationAttemptId.fromString("appattempt_1234_1234_001"), 1); + Token token = new Token<>(tokenId.getBytes(), + "creds".getBytes(Charset.forName("UTF-8")), tokenId.getKind(), + new Text("foo")); + nonDockerCreds.addToken(new Text("amrmtoken"), token); + DockerClientConfigHandler + .writeDockerCredentialsToPath(new File("outfile"), nonDockerCreds); + } } \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java index 40cb031..da46e20 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java @@ -1231,7 +1231,7 @@ private void addDockerClientConfigToRunCommand(ContainerRuntimeContext ctx, if (tokens.hasRemaining()) { try { credentials = DockerClientConfigHandler - .getCredentialsFromTokensByteBuffer(tokens); + .getDockerCredentialsFromTokensByteBuffer(tokens); } catch (IOException e) { throw new ContainerExecutionException("Unable to read tokens."); }