diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java index 60facec..480f8a3 100644 --- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java +++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java @@ -810,7 +810,7 @@ static ContainerLaunchContext createContainerLaunchContext( ContainerLaunchContext container = BuilderUtils.newContainerLaunchContext( commonContainerSpec.getUser(), commonContainerSpec.getLocalResources(), myEnv, commands, - myServiceData, commonContainerSpec.getContainerTokens().duplicate(), + myServiceData, commonContainerSpec.getTokens().duplicate(), applicationACLs); return container; diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttemptContainerRequest.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttemptContainerRequest.java index 54be1d7..6704e00 100644 --- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttemptContainerRequest.java +++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttemptContainerRequest.java @@ -123,7 +123,7 @@ public void testAttemptContainerRequest() throws Exception { Credentials launchCredentials = new Credentials(); DataInputByteBuffer dibb = new DataInputByteBuffer(); - dibb.reset(launchCtx.getContainerTokens()); + dibb.reset(launchCtx.getTokens()); launchCredentials.readTokenStorageStream(dibb); // verify all tokens specified for the task attempt are in the launch context diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerLaunchContext.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerLaunchContext.java index 36cfdfb..da6f6d4 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerLaunchContext.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerLaunchContext.java @@ -68,20 +68,24 @@ void setUser(String user); /** - * Get security tokens (if security is enabled). - * @return security tokens (if security is enabled) + * Get all the tokens needed by this container. It may include file-system + * tokens, ApplicationMaster related tokens if this container is an + * ApplicationMaster or framework level tokens needed by this container to + * communicate to various services in a secure manner. + * + * @return tokens needed by this container. */ @Public @Stable - ByteBuffer getContainerTokens(); + ByteBuffer getTokens(); /** - * Set security tokens (if security is enabled). - * @param containerToken security tokens + * Set security tokens needed by this container. + * @param tokens security tokens */ @Public @Stable - void setContainerTokens(ByteBuffer containerToken); + void setTokens(ByteBuffer tokens); /** * Get LocalResource required by the container. diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerLaunchContextPBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerLaunchContextPBImpl.java index 6a26508..5ab7e12 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerLaunchContextPBImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerLaunchContextPBImpl.java @@ -47,7 +47,7 @@ boolean viaProto = false; private Map localResources = null; - private ByteBuffer containerTokens = null; + private ByteBuffer tokens = null; private Map serviceData = null; private Map environment = null; private List commands = null; @@ -73,8 +73,8 @@ private void mergeLocalToBuilder() { if (this.localResources != null) { addLocalResourcesToProto(); } - if (this.containerTokens != null) { - builder.setContainerTokens(convertToProtoFormat(this.containerTokens)); + if (this.tokens != null) { + builder.setTokens(convertToProtoFormat(this.tokens)); } if (this.serviceData != null) { addServiceDataToProto(); @@ -226,25 +226,25 @@ public boolean hasNext() { } @Override - public ByteBuffer getContainerTokens() { + public ByteBuffer getTokens() { ContainerLaunchContextProtoOrBuilder p = viaProto ? proto : builder; - if (this.containerTokens != null) { - return this.containerTokens; + if (this.tokens != null) { + return this.tokens; } - if (!p.hasContainerTokens()) { + if (!p.hasTokens()) { return null; } - this.containerTokens = convertFromProtoFormat(p.getContainerTokens()); - return this.containerTokens; + this.tokens = convertFromProtoFormat(p.getTokens()); + return this.tokens; } @Override - public void setContainerTokens(ByteBuffer containerTokens) { + public void setTokens(ByteBuffer tokens) { maybeInitBuilder(); - if (containerTokens == null) { - builder.clearContainerTokens(); + if (tokens == null) { + builder.clearTokens(); } - this.containerTokens = containerTokens; + this.tokens = tokens; } @Override diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto index 212f0ec..9e7b3b6 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto @@ -269,7 +269,7 @@ message QueueUserACLInfoProto { message ContainerLaunchContextProto { optional string user = 1; repeated StringLocalResourceMapProto localResources = 2; - optional bytes container_tokens = 3; + optional bytes tokens = 3; repeated StringBytesMapProto service_data = 4; repeated StringStringMapProto environment = 5; repeated string command = 6; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BuilderUtils.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BuilderUtils.java index f09046e..588c975 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BuilderUtils.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BuilderUtils.java @@ -287,7 +287,7 @@ public static ContainerToken newContainerToken(NodeId nodeId, public static ContainerLaunchContext newContainerLaunchContext( String user, Map localResources, Map environment, List commands, - Map serviceData, ByteBuffer containerTokens, + Map serviceData, ByteBuffer tokens, Map acls) { ContainerLaunchContext container = recordFactory .newRecordInstance(ContainerLaunchContext.class); @@ -296,7 +296,7 @@ public static ContainerLaunchContext newContainerLaunchContext( container.setEnvironment(environment); container.setCommands(commands); container.setServiceData(serviceData); - container.setContainerTokens(containerTokens); + container.setTokens(tokens); container.setApplicationACLs(acls); return container; } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java index c79d7c9..4f3ad8c 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java @@ -426,7 +426,7 @@ public StartContainerResponse startContainer(StartContainerRequest request) + launchContext.getUser()); // //////////// Parse credentials - ByteBuffer tokens = launchContext.getContainerTokens(); + ByteBuffer tokens = launchContext.getTokens(); Credentials credentials = new Credentials(); if (tokens != null) { DataInputByteBuffer buf = new DataInputByteBuffer(); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java index 5cc4de1..21db48e 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java @@ -316,7 +316,7 @@ private Credentials parseCredentials(ApplicationSubmissionContext application) throws IOException { Credentials credentials = new Credentials(); DataInputByteBuffer dibb = new DataInputByteBuffer(); - ByteBuffer tokens = application.getAMContainerSpec().getContainerTokens(); + ByteBuffer tokens = application.getAMContainerSpec().getTokens(); if (tokens != null) { dibb.reset(tokens); credentials.readTokenStorageStream(dibb); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java index a9d40eb..3cfb04c 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java @@ -196,9 +196,9 @@ private void setupTokensAndEnv( Credentials credentials = new Credentials(); DataInputByteBuffer dibb = new DataInputByteBuffer(); - if (container.getContainerTokens() != null) { + if (container.getTokens() != null) { // TODO: Don't do this kind of checks everywhere. - dibb.reset(container.getContainerTokens()); + dibb.reset(container.getTokens()); credentials.readTokenStorageStream(dibb); } @@ -219,7 +219,7 @@ private void setupTokensAndEnv( credentials.addToken(appMasterToken.getService(), appMasterToken); DataOutputBuffer dob = new DataOutputBuffer(); credentials.writeTokenStorageToStream(dob); - container.setContainerTokens( + container.setTokens( ByteBuffer.wrap(dob.getData(), 0, dob.getLength())); SecretKey clientSecretKey = diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java index 2259970..7c82bd3 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java @@ -183,7 +183,7 @@ public RMApp submitApp(int masterMemory, String name, String user, DataOutputBuffer dob = new DataOutputBuffer(); ts.writeTokenStorageToStream(dob); ByteBuffer securityTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength()); - clc.setContainerTokens(securityTokens); + clc.setTokens(securityTokens); } sub.setAMContainerSpec(clc); req.setApplicationSubmissionContext(sub); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAMAuthorization.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAMAuthorization.java index bfa2092..23ee9fa 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAMAuthorization.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAMAuthorization.java @@ -78,7 +78,7 @@ public MyContainerManager() { public StartContainerResponse startContainer(StartContainerRequest request) throws YarnRemoteException { - amTokens = request.getContainerLaunchContext().getContainerTokens(); + amTokens = request.getContainerLaunchContext().getTokens(); return null; } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java index fded9fb..a2715b8 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java @@ -468,7 +468,7 @@ public void testTokenRestoredOnRMrestart() throws Exception { ByteBuffer.wrap(dob.getData(), 0, dob.getLength()); Assert.assertEquals(securityTokens, appState .getApplicationSubmissionContext().getAMContainerSpec() - .getContainerTokens()); + .getTokens()); // start new RM MockRM rm2 = new MyMockRM(conf, memStore);