diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java index 51b542a..73719c2 100644 --- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java +++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java @@ -128,7 +128,7 @@ import org.apache.hadoop.yarn.event.Event; import org.apache.hadoop.yarn.event.EventHandler; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; -import org.apache.hadoop.yarn.security.ApplicationTokenIdentifier; +import org.apache.hadoop.yarn.security.AMRMTokenIdentifier; import org.apache.hadoop.yarn.util.Clock; import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.hadoop.yarn.util.SystemClock; @@ -1381,7 +1381,7 @@ protected static void initAndStartAppMaster(final MRAppMaster appMaster, Iterator> iter = credentials.getAllTokens().iterator(); while (iter.hasNext()) { Token token = iter.next(); - if (token.getKind().equals(ApplicationTokenIdentifier.KIND_NAME)) { + if (token.getKind().equals(AMRMTokenIdentifier.KIND_NAME)) { iter.remove(); } } diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestMRAppMaster.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestMRAppMaster.java index 889481f..6d6510f 100644 --- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestMRAppMaster.java +++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestMRAppMaster.java @@ -62,7 +62,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.event.EventHandler; -import org.apache.hadoop.yarn.security.ApplicationTokenIdentifier; +import org.apache.hadoop.yarn.security.AMRMTokenIdentifier; import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.log4j.Level; import org.apache.log4j.LogManager; @@ -346,9 +346,9 @@ public void testMRAppMasterCredentials() throws Exception { credentials.addToken(tokenAlias, myToken); Text appTokenService = new Text("localhost:0"); - Token appToken = - new Token(identifier, password, - ApplicationTokenIdentifier.KIND_NAME, appTokenService); + Token appToken = + new Token(identifier, password, + AMRMTokenIdentifier.KIND_NAME, appTokenService); credentials.addToken(appTokenService, appToken); Text keyAlias = new Text("mySecretKeyAlias"); diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/pipes/TestPipeApplication.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/pipes/TestPipeApplication.java index 7b169d9..04afa15 100644 --- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/pipes/TestPipeApplication.java +++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/pipes/TestPipeApplication.java @@ -61,7 +61,7 @@ import org.apache.hadoop.security.token.Token; import org.apache.hadoop.util.ExitUtil; import org.apache.hadoop.util.Progressable; -import org.apache.hadoop.yarn.security.ApplicationTokenIdentifier; +import org.apache.hadoop.yarn.security.AMRMTokenIdentifier; import org.junit.Assert; import org.junit.Test; @@ -104,7 +104,7 @@ public void testRunner() throws Exception { conf.set(MRJobConfig.CACHE_LOCALFILES, fCommand.getAbsolutePath()); // token for authorization - Token token = new Token( + Token token = new Token( "user".getBytes(), "password".getBytes(), new Text("kind"), new Text( "service")); TokenCache.setJobToken(token, conf.getCredentials()); @@ -168,7 +168,7 @@ public void testApplication() throws Throwable { conf.set(MRJobConfig.CACHE_LOCALFILES, fCommand.getAbsolutePath()); // token for authorization - Token token = new Token( + Token token = new Token( "user".getBytes(), "password".getBytes(), new Text("kind"), new Text( "service")); @@ -389,7 +389,7 @@ public void testPipesReduser() throws Exception { File[] psw = cleanTokenPasswordFile(); JobConf conf = new JobConf(); try { - Token token = new Token( + Token token = new Token( "user".getBytes(), "password".getBytes(), new Text("kind"), new Text( "service")); TokenCache.setJobToken(token, conf.getCredentials()); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index d6cb329..8a15328 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -274,10 +274,10 @@ public static final String DEFAULT_RM_METRICS_RUNTIME_BUCKETS = "60,300,1440"; - public static final String RM_APP_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS = RM_PREFIX - + "application-tokens.master-key-rolling-interval-secs"; + public static final String RM_AMRM_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS = RM_PREFIX + + "am-rm-tokens.master-key-rolling-interval-secs"; - public static final long DEFAULT_RM_APP_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS = + public static final long DEFAULT_RM_AMRM_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS = 24 * 60 * 60; public static final String RM_CONTAINER_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS = diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/AMRMTokenIdentifier.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/AMRMTokenIdentifier.java new file mode 100644 index 0000000..99495d7 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/AMRMTokenIdentifier.java @@ -0,0 +1,103 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +package org.apache.hadoop.yarn.security; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Evolving; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.security.token.TokenIdentifier; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.ApplicationId; + +/** + * AMRMTokenIdentifier is the TokenIdentifier to be used by + * ApplicationMasters to authenticate to the ResourceManager. + */ +@Public +@Evolving +public class AMRMTokenIdentifier extends TokenIdentifier { + + public static final Text KIND_NAME = new Text("YARN_AM_RM_TOKEN"); + + private ApplicationAttemptId applicationAttemptId; + + public AMRMTokenIdentifier() { + } + + public AMRMTokenIdentifier(ApplicationAttemptId appAttemptId) { + this(); + this.applicationAttemptId = appAttemptId; + } + + @Private + public ApplicationAttemptId getApplicationAttemptId() { + return this.applicationAttemptId; + } + + @Override + public void write(DataOutput out) throws IOException { + ApplicationId appId = this.applicationAttemptId.getApplicationId(); + out.writeLong(appId.getClusterTimestamp()); + out.writeInt(appId.getId()); + out.writeInt(this.applicationAttemptId.getAttemptId()); + } + + @Override + public void readFields(DataInput in) throws IOException { + long clusterTimeStamp = in.readLong(); + int appId = in.readInt(); + int attemptId = in.readInt(); + ApplicationId applicationId = + ApplicationId.newInstance(clusterTimeStamp, appId); + this.applicationAttemptId = + ApplicationAttemptId.newInstance(applicationId, attemptId); + } + + @Override + public Text getKind() { + return KIND_NAME; + } + + @Override + public UserGroupInformation getUser() { + if (this.applicationAttemptId == null + || "".equals(this.applicationAttemptId.toString())) { + return null; + } + return UserGroupInformation.createRemoteUser(this.applicationAttemptId + .toString()); + } + + // TODO: Needed? + @InterfaceAudience.Private + public static class Renewer extends Token.TrivialRenewer { + @Override + protected Text getKind() { + return KIND_NAME; + } + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/AMRMTokenSelector.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/AMRMTokenSelector.java new file mode 100644 index 0000000..4693839 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/AMRMTokenSelector.java @@ -0,0 +1,58 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +package org.apache.hadoop.yarn.security; + +import java.util.Collection; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Evolving; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.security.token.TokenIdentifier; +import org.apache.hadoop.security.token.TokenSelector; + +@Public +@Evolving +public class AMRMTokenSelector implements + TokenSelector { + + private static final Log LOG = LogFactory + .getLog(AMRMTokenSelector.class); + + @SuppressWarnings("unchecked") + public Token selectToken(Text service, + Collection> tokens) { + if (service == null) { + return null; + } + LOG.debug("Looking for a token with service " + service.toString()); + for (Token token : tokens) { + LOG.debug("Token kind is " + token.getKind().toString() + + " and the token's service name is " + token.getService()); + if (AMRMTokenIdentifier.KIND_NAME.equals(token.getKind()) + && service.equals(token.getService())) { + return (Token) token; + } + } + return null; + } + +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ApplicationTokenIdentifier.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ApplicationTokenIdentifier.java deleted file mode 100644 index 1007b29..0000000 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ApplicationTokenIdentifier.java +++ /dev/null @@ -1,103 +0,0 @@ -/** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ - -package org.apache.hadoop.yarn.security; - -import java.io.DataInput; -import java.io.DataOutput; -import java.io.IOException; - -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.classification.InterfaceAudience.Private; -import org.apache.hadoop.classification.InterfaceAudience.Public; -import org.apache.hadoop.classification.InterfaceStability.Evolving; -import org.apache.hadoop.io.Text; -import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.security.token.Token; -import org.apache.hadoop.security.token.TokenIdentifier; -import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; -import org.apache.hadoop.yarn.api.records.ApplicationId; - -/** - * ApplicationTokenIdentifier is the TokenIdentifier to be used by - * ApplicationMasters to authenticate to the ResourceManager. - */ -@Public -@Evolving -public class ApplicationTokenIdentifier extends TokenIdentifier { - - public static final Text KIND_NAME = new Text("YARN_APPLICATION_TOKEN"); - - private ApplicationAttemptId applicationAttemptId; - - public ApplicationTokenIdentifier() { - } - - public ApplicationTokenIdentifier(ApplicationAttemptId appAttemptId) { - this(); - this.applicationAttemptId = appAttemptId; - } - - @Private - public ApplicationAttemptId getApplicationAttemptId() { - return this.applicationAttemptId; - } - - @Override - public void write(DataOutput out) throws IOException { - ApplicationId appId = this.applicationAttemptId.getApplicationId(); - out.writeLong(appId.getClusterTimestamp()); - out.writeInt(appId.getId()); - out.writeInt(this.applicationAttemptId.getAttemptId()); - } - - @Override - public void readFields(DataInput in) throws IOException { - long clusterTimeStamp = in.readLong(); - int appId = in.readInt(); - int attemptId = in.readInt(); - ApplicationId applicationId = - ApplicationId.newInstance(clusterTimeStamp, appId); - this.applicationAttemptId = - ApplicationAttemptId.newInstance(applicationId, attemptId); - } - - @Override - public Text getKind() { - return KIND_NAME; - } - - @Override - public UserGroupInformation getUser() { - if (this.applicationAttemptId == null - || "".equals(this.applicationAttemptId.toString())) { - return null; - } - return UserGroupInformation.createRemoteUser(this.applicationAttemptId - .toString()); - } - - // TODO: Needed? - @InterfaceAudience.Private - public static class Renewer extends Token.TrivialRenewer { - @Override - protected Text getKind() { - return KIND_NAME; - } - } -} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ApplicationTokenSelector.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ApplicationTokenSelector.java deleted file mode 100644 index da35091..0000000 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ApplicationTokenSelector.java +++ /dev/null @@ -1,58 +0,0 @@ -/** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ - -package org.apache.hadoop.yarn.security; - -import java.util.Collection; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.classification.InterfaceAudience.Public; -import org.apache.hadoop.classification.InterfaceStability.Evolving; -import org.apache.hadoop.io.Text; -import org.apache.hadoop.security.token.Token; -import org.apache.hadoop.security.token.TokenIdentifier; -import org.apache.hadoop.security.token.TokenSelector; - -@Public -@Evolving -public class ApplicationTokenSelector implements - TokenSelector { - - private static final Log LOG = LogFactory - .getLog(ApplicationTokenSelector.class); - - @SuppressWarnings("unchecked") - public Token selectToken(Text service, - Collection> tokens) { - if (service == null) { - return null; - } - LOG.debug("Looking for a token with service " + service.toString()); - for (Token token : tokens) { - LOG.debug("Token kind is " + token.getKind().toString() - + " and the token's service name is " + token.getService()); - if (ApplicationTokenIdentifier.KIND_NAME.equals(token.getKind()) - && service.equals(token.getService())) { - return (Token) token; - } - } - return null; - } - -} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/SchedulerSecurityInfo.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/SchedulerSecurityInfo.java index 0d62436..bfc34d3 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/SchedulerSecurityInfo.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/SchedulerSecurityInfo.java @@ -54,7 +54,7 @@ public TokenInfo getTokenInfo(Class protocol, Configuration conf) { @Override public Class> value() { - return ApplicationTokenSelector.class; + return AMRMTokenSelector.class; } }; } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier index 2334040..18a9dac 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier @@ -12,6 +12,6 @@ # limitations under the License. # org.apache.hadoop.yarn.security.ContainerTokenIdentifier -org.apache.hadoop.yarn.security.ApplicationTokenIdentifier +org.apache.hadoop.yarn.security.AMRMTokenIdentifier org.apache.hadoop.yarn.security.client.ClientTokenIdentifier org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer index 0e87a7c..dd0b2c4 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer @@ -11,6 +11,6 @@ # See the License for the specific language governing permissions and # limitations under the License. # -org.apache.hadoop.yarn.security.ApplicationTokenIdentifier$Renewer +org.apache.hadoop.yarn.security.AMRMTokenIdentifier$Renewer org.apache.hadoop.yarn.security.ContainerTokenIdentifier$Renewer org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier$Renewer diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java index d9d7e20..b2f3239 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java @@ -118,7 +118,7 @@ protected void serviceStart() throws Exception { this.server = rpc.getServer(ApplicationMasterProtocol.class, this, masterServiceAddress, - conf, this.rmContext.getApplicationTokenSecretManager(), + conf, this.rmContext.getAMRMTokenSecretManager(), conf.getInt(YarnConfiguration.RM_SCHEDULER_CLIENT_THREAD_COUNT, YarnConfiguration.DEFAULT_RM_SCHEDULER_CLIENT_THREAD_COUNT)); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java index 78a4dd6..b29da14 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java @@ -28,7 +28,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.AMLivelinessMonitor; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.ContainerAllocationExpirer; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; -import org.apache.hadoop.yarn.server.resourcemanager.security.ApplicationTokenSecretManager; +import org.apache.hadoop.yarn.server.resourcemanager.security.AMRMTokenSecretManager; import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM; import org.apache.hadoop.yarn.server.resourcemanager.security.DelegationTokenRenewer; import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager; @@ -57,7 +57,7 @@ DelegationTokenRenewer getDelegationTokenRenewer(); - ApplicationTokenSecretManager getApplicationTokenSecretManager(); + AMRMTokenSecretManager getAMRMTokenSecretManager(); RMContainerTokenSecretManager getContainerTokenSecretManager(); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java index 26711f7..f981fb6 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java @@ -31,7 +31,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.AMLivelinessMonitor; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.ContainerAllocationExpirer; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; -import org.apache.hadoop.yarn.server.resourcemanager.security.ApplicationTokenSecretManager; +import org.apache.hadoop.yarn.server.resourcemanager.security.AMRMTokenSecretManager; import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM; import org.apache.hadoop.yarn.server.resourcemanager.security.DelegationTokenRenewer; import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager; @@ -57,7 +57,7 @@ private RMStateStore stateStore = null; private ContainerAllocationExpirer containerAllocationExpirer; private final DelegationTokenRenewer tokenRenewer; - private final ApplicationTokenSecretManager appTokenSecretManager; + private final AMRMTokenSecretManager appTokenSecretManager; private final RMContainerTokenSecretManager containerTokenSecretManager; private final NMTokenSecretManagerInRM nmTokenSecretManager; private final ClientToAMTokenSecretManagerInRM clientToAMTokenSecretManager; @@ -68,7 +68,7 @@ public RMContextImpl(Dispatcher rmDispatcher, AMLivelinessMonitor amLivelinessMonitor, AMLivelinessMonitor amFinishingMonitor, DelegationTokenRenewer tokenRenewer, - ApplicationTokenSecretManager appTokenSecretManager, + AMRMTokenSecretManager appTokenSecretManager, RMContainerTokenSecretManager containerTokenSecretManager, NMTokenSecretManagerInRM nmTokenSecretManager, ClientToAMTokenSecretManagerInRM clientTokenSecretManager) { @@ -91,7 +91,7 @@ public RMContextImpl(Dispatcher rmDispatcher, AMLivelinessMonitor amLivelinessMonitor, AMLivelinessMonitor amFinishingMonitor, DelegationTokenRenewer tokenRenewer, - ApplicationTokenSecretManager appTokenSecretManager, + AMRMTokenSecretManager appTokenSecretManager, RMContainerTokenSecretManager containerTokenSecretManager, NMTokenSecretManagerInRM nmTokenSecretManager, ClientToAMTokenSecretManagerInRM clientTokenSecretManager) { @@ -155,7 +155,7 @@ public DelegationTokenRenewer getDelegationTokenRenewer() { } @Override - public ApplicationTokenSecretManager getApplicationTokenSecretManager() { + public AMRMTokenSecretManager getAMRMTokenSecretManager() { return this.appTokenSecretManager; } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java index c11a982..291f244 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java @@ -67,7 +67,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEventType; -import org.apache.hadoop.yarn.server.resourcemanager.security.ApplicationTokenSecretManager; +import org.apache.hadoop.yarn.server.resourcemanager.security.AMRMTokenSecretManager; import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM; import org.apache.hadoop.yarn.server.resourcemanager.security.DelegationTokenRenewer; import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager; @@ -87,7 +87,7 @@ /** * The ResourceManager is the main class that is a set of components. - * "I am the ResourceManager. All your resources are belong to us..." + * "I am the ResourceManager. All your resources belong to us..." * */ @SuppressWarnings("unchecked") @@ -107,7 +107,7 @@ protected RMContainerTokenSecretManager containerTokenSecretManager; protected NMTokenSecretManagerInRM nmTokenSecretManager; - protected ApplicationTokenSecretManager appTokenSecretManager; + protected AMRMTokenSecretManager amRmTokenSecretManager; private Dispatcher rmDispatcher; @@ -150,7 +150,7 @@ protected void serviceInit(Configuration conf) throws Exception { this.rmDispatcher = createDispatcher(); addIfService(this.rmDispatcher); - this.appTokenSecretManager = createApplicationTokenSecretManager(conf); + this.amRmTokenSecretManager = createAMRMTokenSecretManager(conf); this.containerAllocationExpirer = new ContainerAllocationExpirer( this.rmDispatcher); @@ -193,7 +193,7 @@ protected void serviceInit(Configuration conf) throws Exception { this.rmContext = new RMContextImpl(this.rmDispatcher, rmStore, this.containerAllocationExpirer, amLivelinessMonitor, - amFinishingMonitor, tokenRenewer, this.appTokenSecretManager, + amFinishingMonitor, tokenRenewer, this.amRmTokenSecretManager, this.containerTokenSecretManager, this.nmTokenSecretManager, this.clientToAMSecretManager); @@ -294,9 +294,9 @@ protected void addIfService(Object object) { } } - protected ApplicationTokenSecretManager createApplicationTokenSecretManager( + protected AMRMTokenSecretManager createAMRMTokenSecretManager( Configuration conf) { - return new ApplicationTokenSecretManager(conf); + return new AMRMTokenSecretManager(conf); } protected ResourceScheduler createScheduler() { @@ -557,7 +557,7 @@ protected void serviceStart() throws Exception { throw new YarnRuntimeException("Failed to login", ie); } - this.appTokenSecretManager.start(); + this.amRmTokenSecretManager.start(); this.containerTokenSecretManager.start(); this.nmTokenSecretManager.start(); @@ -617,8 +617,8 @@ protected void serviceStop() throws Exception { rmDTSecretManager.stopThreads(); } - if (appTokenSecretManager != null) { - this.appTokenSecretManager.stop(); + if (amRmTokenSecretManager != null) { + this.amRmTokenSecretManager.stop(); } if (containerTokenSecretManager != null) { this.containerTokenSecretManager.stop(); @@ -731,8 +731,8 @@ public NMTokenSecretManagerInRM getRMNMTokenSecretManager() { } @Private - public ApplicationTokenSecretManager getApplicationTokenSecretManager(){ - return this.appTokenSecretManager; + public AMRMTokenSecretManager getAMRMTokenSecretManager(){ + return this.amRmTokenSecretManager; } @Override diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java index 60258b1..3bbd84b 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java @@ -52,7 +52,7 @@ import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.ipc.YarnRPC; -import org.apache.hadoop.yarn.security.ApplicationTokenIdentifier; +import org.apache.hadoop.yarn.security.AMRMTokenIdentifier; import org.apache.hadoop.yarn.security.ContainerTokenIdentifier; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; @@ -201,10 +201,10 @@ private void setupTokensAndEnv( } // Add application token - Token applicationToken = - application.getApplicationToken(); - if(applicationToken != null) { - credentials.addToken(applicationToken.getService(), applicationToken); + Token amrmToken = + application.getAMRMToken(); + if(amrmToken != null) { + credentials.addToken(amrmToken.getService(), amrmToken); } DataOutputBuffer dob = new DataOutputBuffer(); credentials.writeTokenStorageToStream(dob); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java index 0665022..a0b5499 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java @@ -41,7 +41,7 @@ import org.apache.hadoop.yarn.event.AsyncDispatcher; import org.apache.hadoop.yarn.event.Dispatcher; import org.apache.hadoop.yarn.event.EventHandler; -import org.apache.hadoop.yarn.security.ApplicationTokenIdentifier; +import org.apache.hadoop.yarn.security.AMRMTokenIdentifier; import org.apache.hadoop.yarn.security.client.ClientTokenIdentifier; import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.ApplicationAttemptStateDataPBImpl; import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.ApplicationStateDataPBImpl; @@ -378,7 +378,7 @@ protected abstract void removeApplicationState(ApplicationState appState) private Credentials getTokensFromAppAttempt(RMAppAttempt appAttempt) { Credentials credentials = new Credentials(); - Token appToken = appAttempt.getApplicationToken(); + Token appToken = appAttempt.getAMRMToken(); if(appToken != null){ credentials.addToken(appToken.getService(), appToken); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttempt.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttempt.java index b9c7eb2..d4795b4 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttempt.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttempt.java @@ -31,7 +31,7 @@ import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.EventHandler; -import org.apache.hadoop.yarn.security.ApplicationTokenIdentifier; +import org.apache.hadoop.yarn.security.AMRMTokenIdentifier; import org.apache.hadoop.yarn.security.client.ClientTokenIdentifier; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; @@ -149,10 +149,10 @@ ApplicationSubmissionContext getSubmissionContext(); /** - * The application token belonging to this app attempt - * @return The application token belonging to this app attempt + * The AMRMToken belonging to this app attempt + * @return The AMRMToken belonging to this app attempt */ - Token getApplicationToken(); + Token getAMRMToken(); /** * Get application container and resource usage information. diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java index bb2e252..d249cfb 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java @@ -61,8 +61,8 @@ import org.apache.hadoop.yarn.event.EventHandler; import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; -import org.apache.hadoop.yarn.security.ApplicationTokenIdentifier; -import org.apache.hadoop.yarn.security.ApplicationTokenSelector; +import org.apache.hadoop.yarn.security.AMRMTokenIdentifier; +import org.apache.hadoop.yarn.security.AMRMTokenSelector; import org.apache.hadoop.yarn.security.client.ClientTokenIdentifier; import org.apache.hadoop.yarn.security.client.ClientTokenSelector; import org.apache.hadoop.yarn.server.resourcemanager.ApplicationMasterService; @@ -131,7 +131,7 @@ private final ApplicationAttemptId applicationAttemptId; private Token clientToken; private final ApplicationSubmissionContext submissionContext; - private Token applicationToken = null; + private Token amrmToken = null; //nodes on while this attempt's containers ran private final Set ranNodes = @@ -503,8 +503,8 @@ private void setTrackingUrlToRMAppPage() { } @Override - public Token getApplicationToken() { - return this.applicationToken; + public Token getAMRMToken() { + return this.amrmToken; } @Override @@ -682,14 +682,14 @@ private void recoverAppAttemptTokens(Credentials appAttemptTokens) { YarnConfiguration.RM_SCHEDULER_ADDRESS, YarnConfiguration.DEFAULT_RM_SCHEDULER_ADDRESS, YarnConfiguration.DEFAULT_RM_SCHEDULER_PORT); - ApplicationTokenSelector appTokenSelector = new ApplicationTokenSelector(); - this.applicationToken = + AMRMTokenSelector appTokenSelector = new AMRMTokenSelector(); + this.amrmToken = appTokenSelector.selectToken( SecurityUtil.buildTokenService(serviceAddr), appAttemptTokens.getAllTokens()); // For now, no need to populate tokens back to - // ApplicationTokenSecretManager, because running attempts are rebooted + // AMRMTokenSecretManager, because running attempts are rebooted // Later in work-preserve restart, we'll create NEW->RUNNING transition // in which the restored tokens will be added to the secret manager } @@ -727,11 +727,11 @@ public void transition(RMAppAttemptImpl appAttempt, appAttempt.rmContext.getClientToAMTokenSecretManager()); // create application token - ApplicationTokenIdentifier id = - new ApplicationTokenIdentifier(appAttempt.applicationAttemptId); - Token applicationToken = - new Token(id, - appAttempt.rmContext.getApplicationTokenSecretManager()); + AMRMTokenIdentifier id = + new AMRMTokenIdentifier(appAttempt.applicationAttemptId); + Token amRmToken = + new Token(id, + appAttempt.rmContext.getAMRMTokenSecretManager()); InetSocketAddress serviceAddr = appAttempt.conf.getSocketAddr( YarnConfiguration.RM_SCHEDULER_ADDRESS, @@ -739,9 +739,9 @@ public void transition(RMAppAttemptImpl appAttempt, YarnConfiguration.DEFAULT_RM_SCHEDULER_PORT); // normally the client should set the service after acquiring the // token, but this token is directly provided to the AMs - SecurityUtil.setTokenService(applicationToken, serviceAddr); + SecurityUtil.setTokenService(amRmToken, serviceAddr); - appAttempt.applicationToken = applicationToken; + appAttempt.amrmToken = amRmToken; } @@ -902,8 +902,8 @@ public void transition(RMAppAttemptImpl appAttempt, appAttempt.eventHandler.handle(new AppRemovedSchedulerEvent(appAttemptId, finalAttemptState)); - // Remove the AppAttempt from the ApplicationTokenSecretManager - appAttempt.rmContext.getApplicationTokenSecretManager() + // Remove the AppAttempt from the AMRMTokenSecretManager + appAttempt.rmContext.getAMRMTokenSecretManager() .applicationMasterFinished(appAttemptId); } } @@ -1125,8 +1125,8 @@ public RMAppAttemptState transition(RMAppAttemptImpl appAttempt, appAttempt.rmContext.getAMLivelinessMonitor().unregister(appAttemptId); - // Remove the AppAttempt from the ApplicationTokenSecretManager - appAttempt.rmContext.getApplicationTokenSecretManager() + // Remove the AppAttempt from the AMRMTokenSecretManager + appAttempt.rmContext.getAMRMTokenSecretManager() .applicationMasterFinished(appAttemptId); appAttempt.progress = 1.0f; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/AMRMTokenSecretManager.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/AMRMTokenSecretManager.java new file mode 100644 index 0000000..3f50e1b --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/AMRMTokenSecretManager.java @@ -0,0 +1,155 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +package org.apache.hadoop.yarn.server.resourcemanager.security; + +import java.util.HashMap; +import java.util.Map; +import java.util.Timer; +import java.util.TimerTask; + +import javax.crypto.SecretKey; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.token.SecretManager; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.security.AMRMTokenIdentifier; + +/** + * AMRM-tokens are per ApplicationAttempt. If users redistribute their + * tokens, it is their headache, god save them. I mean you are not supposed to + * distribute keys to your vault, right? Anyways, ResourceManager saves each + * token locally in memory till application finishes and to a store for restart, + * so no need to remember master-keys even after rolling them. + */ +public class AMRMTokenSecretManager extends + SecretManager { + + private static final Log LOG = LogFactory + .getLog(AMRMTokenSecretManager.class); + + private SecretKey masterKey; + private final Timer timer; + private final long rollingInterval; + + private final Map passwords = + new HashMap(); + + /** + * Create an {@link AMRMTokenSecretManager} + */ + public AMRMTokenSecretManager(Configuration conf) { + rollMasterKey(); + this.timer = new Timer(); + this.rollingInterval = + conf + .getLong( + YarnConfiguration.RM_AMRM_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS, + YarnConfiguration.DEFAULT_RM_AMRM_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS) * 1000; + } + + public void start() { + this.timer.scheduleAtFixedRate(new MasterKeyRoller(), 0, rollingInterval); + } + + public void stop() { + this.timer.cancel(); + } + + public synchronized void applicationMasterFinished( + ApplicationAttemptId appAttemptId) { + if (LOG.isDebugEnabled()) { + LOG.debug("Application finished, removing password for " + appAttemptId); + } + this.passwords.remove(appAttemptId); + } + + private class MasterKeyRoller extends TimerTask { + @Override + public void run() { + rollMasterKey(); + } + } + + @Private + public synchronized void setMasterKey(SecretKey masterKey) { + this.masterKey = masterKey; + } + + @Private + public synchronized SecretKey getMasterKey() { + return this.masterKey; + } + + @Private + synchronized void rollMasterKey() { + LOG.info("Rolling master-key for amrm-tokens"); + this.masterKey = generateSecret(); + } + + /** + * Create a password for a given {@link AMRMTokenIdentifier}. Used to + * send to the AppicationAttempt which can give it back during authentication. + */ + @Override + public synchronized byte[] createPassword( + AMRMTokenIdentifier identifier) { + ApplicationAttemptId applicationAttemptId = + identifier.getApplicationAttemptId(); + if (LOG.isDebugEnabled()) { + LOG.debug("Creating password for " + applicationAttemptId); + } + byte[] password = createPassword(identifier.getBytes(), masterKey); + this.passwords.put(applicationAttemptId, password); + return password; + } + + /** + * Retrieve the password for the given {@link AMRMTokenIdentifier}. + * Used by RPC layer to validate a remote {@link AMRMTokenIdentifier}. + */ + @Override + public synchronized byte[] retrievePassword( + AMRMTokenIdentifier identifier) throws InvalidToken { + ApplicationAttemptId applicationAttemptId = + identifier.getApplicationAttemptId(); + if (LOG.isDebugEnabled()) { + LOG.debug("Trying to retrieve password for " + applicationAttemptId); + } + byte[] password = this.passwords.get(applicationAttemptId); + if (password == null) { + throw new InvalidToken("Password not found for ApplicationAttempt " + + applicationAttemptId); + } + return password; + } + + /** + * Creates an empty TokenId to be used for de-serializing an + * {@link AMRMTokenIdentifier} by the RPC layer. + */ + @Override + public AMRMTokenIdentifier createIdentifier() { + return new AMRMTokenIdentifier(); + } + +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/ApplicationTokenSecretManager.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/ApplicationTokenSecretManager.java deleted file mode 100644 index 8a65c09..0000000 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/ApplicationTokenSecretManager.java +++ /dev/null @@ -1,155 +0,0 @@ -/** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ - -package org.apache.hadoop.yarn.server.resourcemanager.security; - -import java.util.HashMap; -import java.util.Map; -import java.util.Timer; -import java.util.TimerTask; - -import javax.crypto.SecretKey; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.classification.InterfaceAudience.Private; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.security.token.SecretManager; -import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; -import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.hadoop.yarn.security.ApplicationTokenIdentifier; - -/** - * Application-tokens are per ApplicationAttempt. If users redistribute their - * tokens, it is their headache, god save them. I mean you are not supposed to - * distribute keys to your vault, right? Anyways, ResourceManager saves each - * token locally in memory till application finishes and to a store for restart, - * so no need to remember master-keys even after rolling them. - */ -public class ApplicationTokenSecretManager extends - SecretManager { - - private static final Log LOG = LogFactory - .getLog(ApplicationTokenSecretManager.class); - - private SecretKey masterKey; - private final Timer timer; - private final long rollingInterval; - - private final Map passwords = - new HashMap(); - - /** - * Create an {@link ApplicationTokenSecretManager} - */ - public ApplicationTokenSecretManager(Configuration conf) { - rollMasterKey(); - this.timer = new Timer(); - this.rollingInterval = - conf - .getLong( - YarnConfiguration.RM_APP_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS, - YarnConfiguration.DEFAULT_RM_APP_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS) * 1000; - } - - public void start() { - this.timer.scheduleAtFixedRate(new MasterKeyRoller(), 0, rollingInterval); - } - - public void stop() { - this.timer.cancel(); - } - - public synchronized void applicationMasterFinished( - ApplicationAttemptId appAttemptId) { - if (LOG.isDebugEnabled()) { - LOG.debug("Application finished, removing password for " + appAttemptId); - } - this.passwords.remove(appAttemptId); - } - - private class MasterKeyRoller extends TimerTask { - @Override - public void run() { - rollMasterKey(); - } - } - - @Private - public synchronized void setMasterKey(SecretKey masterKey) { - this.masterKey = masterKey; - } - - @Private - public synchronized SecretKey getMasterKey() { - return this.masterKey; - } - - @Private - synchronized void rollMasterKey() { - LOG.info("Rolling master-key for application-tokens"); - this.masterKey = generateSecret(); - } - - /** - * Create a password for a given {@link ApplicationTokenIdentifier}. Used to - * send to the AppicationAttempt which can give it back during authentication. - */ - @Override - public synchronized byte[] createPassword( - ApplicationTokenIdentifier identifier) { - ApplicationAttemptId applicationAttemptId = - identifier.getApplicationAttemptId(); - if (LOG.isDebugEnabled()) { - LOG.debug("Creating password for " + applicationAttemptId); - } - byte[] password = createPassword(identifier.getBytes(), masterKey); - this.passwords.put(applicationAttemptId, password); - return password; - } - - /** - * Retrieve the password for the given {@link ApplicationTokenIdentifier}. - * Used by RPC layer to validate a remote {@link ApplicationTokenIdentifier}. - */ - @Override - public synchronized byte[] retrievePassword( - ApplicationTokenIdentifier identifier) throws InvalidToken { - ApplicationAttemptId applicationAttemptId = - identifier.getApplicationAttemptId(); - if (LOG.isDebugEnabled()) { - LOG.debug("Trying to retrieve password for " + applicationAttemptId); - } - byte[] password = this.passwords.get(applicationAttemptId); - if (password == null) { - throw new InvalidToken("Password not found for ApplicationAttempt " - + applicationAttemptId); - } - return password; - } - - /** - * Creates an empty TokenId to be used for de-serializing an - * {@link ApplicationTokenIdentifier} by the RPC layer. - */ - @Override - public ApplicationTokenIdentifier createIdentifier() { - return new ApplicationTokenIdentifier(); - } - -} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java index b44d376..f585a33 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java @@ -536,7 +536,7 @@ public void testAppAttemptTokensRestoredOnRMRestart() throws Exception { // the appToken and clientToken that are generated when RMAppAttempt is created, HashSet> tokenSet = new HashSet>(); - tokenSet.add(attempt1.getApplicationToken()); + tokenSet.add(attempt1.getAMRMToken()); tokenSet.add(attempt1.getClientToken()); // assert application Token is saved @@ -555,7 +555,7 @@ public void testAppAttemptTokensRestoredOnRMRestart() throws Exception { // assert loaded attempt recovered attempt tokens Assert.assertNotNull(loadedAttempt1); savedTokens.clear(); - savedTokens.add(loadedAttempt1.getApplicationToken()); + savedTokens.add(loadedAttempt1.getAMRMToken()); savedTokens.add(loadedAttempt1.getClientToken()); Assert.assertEquals(tokenSet, savedTokens); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMLaunchFailure.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMLaunchFailure.java index 6f5e1b2..ce3f9b2 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMLaunchFailure.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMLaunchFailure.java @@ -51,7 +51,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.resource.Resources; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler; -import org.apache.hadoop.yarn.server.resourcemanager.security.ApplicationTokenSecretManager; +import org.apache.hadoop.yarn.server.resourcemanager.security.AMRMTokenSecretManager; import org.junit.After; import org.junit.Before; import org.junit.Test; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java index 7181705..72b83af 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java @@ -57,7 +57,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler; -import org.apache.hadoop.yarn.server.resourcemanager.security.ApplicationTokenSecretManager; +import org.apache.hadoop.yarn.server.resourcemanager.security.AMRMTokenSecretManager; import org.apache.hadoop.yarn.server.security.BaseContainerTokenSecretManager; import org.junit.After; import org.junit.Before; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestRMStateStore.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestRMStateStore.java index 75b5d9f..79e4089 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestRMStateStore.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestRMStateStore.java @@ -54,7 +54,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.Dispatcher; import org.apache.hadoop.yarn.event.EventHandler; -import org.apache.hadoop.yarn.security.ApplicationTokenIdentifier; +import org.apache.hadoop.yarn.security.AMRMTokenIdentifier; import org.apache.hadoop.yarn.security.client.ClientTokenIdentifier; import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier; import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.ApplicationAttemptState; @@ -64,7 +64,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptStoredEvent; -import org.apache.hadoop.yarn.server.resourcemanager.security.ApplicationTokenSecretManager; +import org.apache.hadoop.yarn.server.resourcemanager.security.AMRMTokenSecretManager; import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM; import org.apache.hadoop.yarn.util.ConverterUtils; import org.junit.Test; @@ -205,7 +205,7 @@ void storeApp(RMStateStore store, ApplicationId appId, long time) } ContainerId storeAttempt(RMStateStore store, ApplicationAttemptId attemptId, - String containerIdStr, Token appToken, + String containerIdStr, Token appToken, Token clientToken, TestDispatcher dispatcher) throws Exception { @@ -214,7 +214,7 @@ ContainerId storeAttempt(RMStateStore store, ApplicationAttemptId attemptId, RMAppAttempt mockAttempt = mock(RMAppAttempt.class); when(mockAttempt.getAppAttemptId()).thenReturn(attemptId); when(mockAttempt.getMasterContainer()).thenReturn(container); - when(mockAttempt.getApplicationToken()).thenReturn(appToken); + when(mockAttempt.getAMRMToken()).thenReturn(appToken); when(mockAttempt.getClientToken()).thenReturn(clientToken); dispatcher.attemptId = attemptId; dispatcher.storedException = null; @@ -231,8 +231,8 @@ void testRMAppStateStore(RMStateStoreHelper stateStoreHelper) throws Exception { TestDispatcher dispatcher = new TestDispatcher(); store.setDispatcher(dispatcher); - ApplicationTokenSecretManager appTokenMgr = - new ApplicationTokenSecretManager(conf); + AMRMTokenSecretManager appTokenMgr = + new AMRMTokenSecretManager(conf); ClientToAMTokenSecretManagerInRM clientTokenMgr = new ClientToAMTokenSecretManagerInRM(); @@ -249,7 +249,7 @@ void testRMAppStateStore(RMStateStoreHelper stateStoreHelper) throws Exception { ContainerId containerId1 = storeAttempt(store, attemptId1, "container_1352994193343_0001_01_000001", - (Token) (appAttemptToken1.get(0)), + (Token) (appAttemptToken1.get(0)), (Token)(appAttemptToken1.get(1)), dispatcher); @@ -265,7 +265,7 @@ void testRMAppStateStore(RMStateStoreHelper stateStoreHelper) throws Exception { ContainerId containerId2 = storeAttempt(store, attemptId2, "container_1352994193343_0001_02_000001", - (Token) (appAttemptToken2.get(0)), + (Token) (appAttemptToken2.get(0)), (Token)(appAttemptToken2.get(1)), dispatcher); @@ -372,12 +372,12 @@ public void testRMDTSecretManagerStateStore( } private List> generateTokens(ApplicationAttemptId attemptId, - ApplicationTokenSecretManager appTokenMgr, + AMRMTokenSecretManager appTokenMgr, ClientToAMTokenSecretManagerInRM clientTokenMgr, Configuration conf) { - ApplicationTokenIdentifier appTokenId = - new ApplicationTokenIdentifier(attemptId); - Token appToken = - new Token(appTokenId, appTokenMgr); + AMRMTokenIdentifier appTokenId = + new AMRMTokenIdentifier(attemptId); + Token appToken = + new Token(appTokenId, appTokenMgr); appToken.setService(new Text("appToken service")); ClientTokenIdentifier clientTokenId = new ClientTokenIdentifier(attemptId); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java index fc6378f..a4b7404 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java @@ -51,7 +51,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEventType; -import org.apache.hadoop.yarn.server.resourcemanager.security.ApplicationTokenSecretManager; +import org.apache.hadoop.yarn.server.resourcemanager.security.AMRMTokenSecretManager; import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM; import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager; import org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM; @@ -145,7 +145,7 @@ public void setUp() throws Exception { this.rmContext = new RMContextImpl(rmDispatcher, store, containerAllocationExpirer, amLivelinessMonitor, amFinishingMonitor, - null, new ApplicationTokenSecretManager(conf), + null, new AMRMTokenSecretManager(conf), new RMContainerTokenSecretManager(conf), new NMTokenSecretManagerInRM(conf), new ClientToAMTokenSecretManagerInRM()); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java index b9a4108..a394110 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java @@ -75,7 +75,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEventType; -import org.apache.hadoop.yarn.server.resourcemanager.security.ApplicationTokenSecretManager; +import org.apache.hadoop.yarn.server.resourcemanager.security.AMRMTokenSecretManager; import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM; import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager; import org.apache.hadoop.yarn.server.utils.BuilderUtils; @@ -167,7 +167,7 @@ public void setUp() throws Exception { rmContext = new RMContextImpl(rmDispatcher, containerAllocationExpirer, amLivelinessMonitor, amFinishingMonitor, - null, new ApplicationTokenSecretManager(conf), + null, new AMRMTokenSecretManager(conf), new RMContainerTokenSecretManager(conf), new NMTokenSecretManagerInRM(conf), new ClientToAMTokenSecretManagerInRM()); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java index 591b0f5..b15169b 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java @@ -46,7 +46,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode; -import org.apache.hadoop.yarn.server.resourcemanager.security.ApplicationTokenSecretManager; +import org.apache.hadoop.yarn.server.resourcemanager.security.AMRMTokenSecretManager; import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM; import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager; import org.apache.hadoop.yarn.server.utils.BuilderUtils; @@ -86,7 +86,7 @@ public EventHandler getEventHandler() { Configuration conf = new Configuration(); RMContext rmContext = new RMContextImpl(nullDispatcher, cae, null, null, null, - new ApplicationTokenSecretManager(conf), + new AMRMTokenSecretManager(conf), new RMContainerTokenSecretManager(conf), new NMTokenSecretManagerInRM(conf), new ClientToAMTokenSecretManagerInRM()); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestAMRMTokens.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestAMRMTokens.java new file mode 100644 index 0000000..b39aaec --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestAMRMTokens.java @@ -0,0 +1,249 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.security; + +import java.security.PrivilegedAction; + +import javax.crypto.SecretKey; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.CommonConfigurationKeysPublic; +import org.apache.hadoop.io.DataInputByteBuffer; +import org.apache.hadoop.security.Credentials; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.yarn.api.ApplicationMasterProtocol; +import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest; +import org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterRequest; +import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRequest; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; +import org.apache.hadoop.yarn.ipc.YarnRPC; +import org.apache.hadoop.yarn.server.resourcemanager.MockNM; +import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.TestAMAuthorization.MockRMWithAMS; +import org.apache.hadoop.yarn.server.resourcemanager.TestAMAuthorization.MyContainerManager; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; +import org.apache.hadoop.yarn.server.utils.BuilderUtils; +import org.apache.hadoop.yarn.util.Records; +import org.junit.Assert; +import org.junit.Test; + +public class TestAMRMTokens { + + private static final Log LOG = LogFactory.getLog(TestAMRMTokens.class); + + private static final Configuration confWithSecurityEnabled = + new Configuration(); + static { + confWithSecurityEnabled.set( + CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos"); + UserGroupInformation.setConfiguration(confWithSecurityEnabled); + } + + /** + * Validate that application tokens are unusable after the + * application-finishes. + * + * @throws Exception + */ + @Test + public void testTokenExpiry() throws Exception { + + MyContainerManager containerManager = new MyContainerManager(); + final MockRM rm = + new MockRMWithAMS(confWithSecurityEnabled, containerManager); + rm.start(); + + final Configuration conf = rm.getConfig(); + final YarnRPC rpc = YarnRPC.create(conf); + ApplicationMasterProtocol rmClient = null; + + try { + MockNM nm1 = rm.registerNode("localhost:1234", 5120); + + RMApp app = rm.submitApp(1024); + + nm1.nodeHeartbeat(true); + + int waitCount = 0; + while (containerManager.amTokens == null && waitCount++ < 20) { + LOG.info("Waiting for AM Launch to happen.."); + Thread.sleep(1000); + } + Assert.assertNotNull(containerManager.amTokens); + + RMAppAttempt attempt = app.getCurrentAppAttempt(); + ApplicationAttemptId applicationAttemptId = attempt.getAppAttemptId(); + + // Create a client to the RM. + UserGroupInformation currentUser = + UserGroupInformation + .createRemoteUser(applicationAttemptId.toString()); + Credentials credentials = new Credentials(); + DataInputByteBuffer buf = new DataInputByteBuffer(); + containerManager.amTokens.rewind(); + buf.reset(containerManager.amTokens); + credentials.readTokenStorageStream(buf); + currentUser.addCredentials(credentials); + + rmClient = createRMClient(rm, conf, rpc, currentUser); + + RegisterApplicationMasterRequest request = + Records.newRecord(RegisterApplicationMasterRequest.class); + request.setApplicationAttemptId(applicationAttemptId); + rmClient.registerApplicationMaster(request); + + FinishApplicationMasterRequest finishAMRequest = + Records.newRecord(FinishApplicationMasterRequest.class); + finishAMRequest.setAppAttemptId(applicationAttemptId); + finishAMRequest + .setFinalApplicationStatus(FinalApplicationStatus.SUCCEEDED); + finishAMRequest.setDiagnostics("diagnostics"); + finishAMRequest.setTrackingUrl("url"); + rmClient.finishApplicationMaster(finishAMRequest); + + // Now simulate trying to allocate. RPC call itself should throw auth + // exception. + rpc.stopProxy(rmClient, conf); // To avoid using cached client + rmClient = createRMClient(rm, conf, rpc, currentUser); + request.setApplicationAttemptId(BuilderUtils.newApplicationAttemptId( + BuilderUtils.newApplicationId(12345, 78), 987)); + AllocateRequest allocateRequest = + Records.newRecord(AllocateRequest.class); + allocateRequest.setApplicationAttemptId(applicationAttemptId); + try { + rmClient.allocate(allocateRequest); + Assert.fail("You got to be kidding me! " + + "Using App tokens after app-finish should fail!"); + } catch (Throwable t) { + LOG.info("Exception found is ", t); + // The exception will still have the earlier appAttemptId as it picks it + // up from the token. + Assert.assertTrue(t.getCause().getMessage().contains( + "Password not found for ApplicationAttempt " + + applicationAttemptId.toString())); + } + + } finally { + rm.stop(); + if (rmClient != null) { + rpc.stopProxy(rmClient, conf); // To avoid using cached client + } + } + } + + /** + * Validate master-key-roll-over and that tokens are usable even after + * master-key-roll-over. + * + * @throws Exception + */ + @Test + public void testMasterKeyRollOver() throws Exception { + + MyContainerManager containerManager = new MyContainerManager(); + final MockRM rm = + new MockRMWithAMS(confWithSecurityEnabled, containerManager); + rm.start(); + + final Configuration conf = rm.getConfig(); + final YarnRPC rpc = YarnRPC.create(conf); + ApplicationMasterProtocol rmClient = null; + + try { + MockNM nm1 = rm.registerNode("localhost:1234", 5120); + + RMApp app = rm.submitApp(1024); + + nm1.nodeHeartbeat(true); + + int waitCount = 0; + while (containerManager.amTokens == null && waitCount++ < 20) { + LOG.info("Waiting for AM Launch to happen.."); + Thread.sleep(1000); + } + Assert.assertNotNull(containerManager.amTokens); + + RMAppAttempt attempt = app.getCurrentAppAttempt(); + ApplicationAttemptId applicationAttemptId = attempt.getAppAttemptId(); + + // Create a client to the RM. + UserGroupInformation currentUser = + UserGroupInformation + .createRemoteUser(applicationAttemptId.toString()); + Credentials credentials = new Credentials(); + DataInputByteBuffer buf = new DataInputByteBuffer(); + containerManager.amTokens.rewind(); + buf.reset(containerManager.amTokens); + credentials.readTokenStorageStream(buf); + currentUser.addCredentials(credentials); + + rmClient = createRMClient(rm, conf, rpc, currentUser); + + RegisterApplicationMasterRequest request = + Records.newRecord(RegisterApplicationMasterRequest.class); + request.setApplicationAttemptId(applicationAttemptId); + rmClient.registerApplicationMaster(request); + + // One allocate call. + AllocateRequest allocateRequest = + Records.newRecord(AllocateRequest.class); + allocateRequest.setApplicationAttemptId(applicationAttemptId); + Assert.assertTrue( + rmClient.allocate(allocateRequest).getAMCommand() == null); + + // Simulate a master-key-roll-over + AMRMTokenSecretManager appTokenSecretManager = + rm.getRMContext().getAMRMTokenSecretManager(); + SecretKey oldKey = appTokenSecretManager.getMasterKey(); + appTokenSecretManager.rollMasterKey(); + SecretKey newKey = appTokenSecretManager.getMasterKey(); + Assert.assertFalse("Master key should have changed!", + oldKey.equals(newKey)); + + // Another allocate call. Should continue to work. + rpc.stopProxy(rmClient, conf); // To avoid using cached client + rmClient = createRMClient(rm, conf, rpc, currentUser); + allocateRequest = Records.newRecord(AllocateRequest.class); + allocateRequest.setApplicationAttemptId(applicationAttemptId); + Assert.assertTrue( + rmClient.allocate(allocateRequest).getAMCommand() == null); + } finally { + rm.stop(); + if (rmClient != null) { + rpc.stopProxy(rmClient, conf); // To avoid using cached client + } + } + } + + private ApplicationMasterProtocol createRMClient(final MockRM rm, + final Configuration conf, final YarnRPC rpc, + UserGroupInformation currentUser) { + return currentUser.doAs(new PrivilegedAction() { + @Override + public ApplicationMasterProtocol run() { + return (ApplicationMasterProtocol) rpc.getProxy(ApplicationMasterProtocol.class, rm + .getApplicationMasterService().getBindAddress(), conf); + } + }); + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestApplicationTokens.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestApplicationTokens.java deleted file mode 100644 index 3e60462..0000000 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestApplicationTokens.java +++ /dev/null @@ -1,249 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.server.resourcemanager.security; - -import java.security.PrivilegedAction; - -import javax.crypto.SecretKey; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.CommonConfigurationKeysPublic; -import org.apache.hadoop.io.DataInputByteBuffer; -import org.apache.hadoop.security.Credentials; -import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.yarn.api.ApplicationMasterProtocol; -import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest; -import org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterRequest; -import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRequest; -import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; -import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; -import org.apache.hadoop.yarn.ipc.YarnRPC; -import org.apache.hadoop.yarn.server.resourcemanager.MockNM; -import org.apache.hadoop.yarn.server.resourcemanager.MockRM; -import org.apache.hadoop.yarn.server.resourcemanager.TestAMAuthorization.MockRMWithAMS; -import org.apache.hadoop.yarn.server.resourcemanager.TestAMAuthorization.MyContainerManager; -import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; -import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; -import org.apache.hadoop.yarn.server.utils.BuilderUtils; -import org.apache.hadoop.yarn.util.Records; -import org.junit.Assert; -import org.junit.Test; - -public class TestApplicationTokens { - - private static final Log LOG = LogFactory.getLog(TestApplicationTokens.class); - - private static final Configuration confWithSecurityEnabled = - new Configuration(); - static { - confWithSecurityEnabled.set( - CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos"); - UserGroupInformation.setConfiguration(confWithSecurityEnabled); - } - - /** - * Validate that application tokens are unusable after the - * application-finishes. - * - * @throws Exception - */ - @Test - public void testTokenExpiry() throws Exception { - - MyContainerManager containerManager = new MyContainerManager(); - final MockRM rm = - new MockRMWithAMS(confWithSecurityEnabled, containerManager); - rm.start(); - - final Configuration conf = rm.getConfig(); - final YarnRPC rpc = YarnRPC.create(conf); - ApplicationMasterProtocol rmClient = null; - - try { - MockNM nm1 = rm.registerNode("localhost:1234", 5120); - - RMApp app = rm.submitApp(1024); - - nm1.nodeHeartbeat(true); - - int waitCount = 0; - while (containerManager.amTokens == null && waitCount++ < 20) { - LOG.info("Waiting for AM Launch to happen.."); - Thread.sleep(1000); - } - Assert.assertNotNull(containerManager.amTokens); - - RMAppAttempt attempt = app.getCurrentAppAttempt(); - ApplicationAttemptId applicationAttemptId = attempt.getAppAttemptId(); - - // Create a client to the RM. - UserGroupInformation currentUser = - UserGroupInformation - .createRemoteUser(applicationAttemptId.toString()); - Credentials credentials = new Credentials(); - DataInputByteBuffer buf = new DataInputByteBuffer(); - containerManager.amTokens.rewind(); - buf.reset(containerManager.amTokens); - credentials.readTokenStorageStream(buf); - currentUser.addCredentials(credentials); - - rmClient = createRMClient(rm, conf, rpc, currentUser); - - RegisterApplicationMasterRequest request = - Records.newRecord(RegisterApplicationMasterRequest.class); - request.setApplicationAttemptId(applicationAttemptId); - rmClient.registerApplicationMaster(request); - - FinishApplicationMasterRequest finishAMRequest = - Records.newRecord(FinishApplicationMasterRequest.class); - finishAMRequest.setAppAttemptId(applicationAttemptId); - finishAMRequest - .setFinalApplicationStatus(FinalApplicationStatus.SUCCEEDED); - finishAMRequest.setDiagnostics("diagnostics"); - finishAMRequest.setTrackingUrl("url"); - rmClient.finishApplicationMaster(finishAMRequest); - - // Now simulate trying to allocate. RPC call itself should throw auth - // exception. - rpc.stopProxy(rmClient, conf); // To avoid using cached client - rmClient = createRMClient(rm, conf, rpc, currentUser); - request.setApplicationAttemptId(BuilderUtils.newApplicationAttemptId( - BuilderUtils.newApplicationId(12345, 78), 987)); - AllocateRequest allocateRequest = - Records.newRecord(AllocateRequest.class); - allocateRequest.setApplicationAttemptId(applicationAttemptId); - try { - rmClient.allocate(allocateRequest); - Assert.fail("You got to be kidding me! " - + "Using App tokens after app-finish should fail!"); - } catch (Throwable t) { - LOG.info("Exception found is ", t); - // The exception will still have the earlier appAttemptId as it picks it - // up from the token. - Assert.assertTrue(t.getCause().getMessage().contains( - "Password not found for ApplicationAttempt " + - applicationAttemptId.toString())); - } - - } finally { - rm.stop(); - if (rmClient != null) { - rpc.stopProxy(rmClient, conf); // To avoid using cached client - } - } - } - - /** - * Validate master-key-roll-over and that tokens are usable even after - * master-key-roll-over. - * - * @throws Exception - */ - @Test - public void testMasterKeyRollOver() throws Exception { - - MyContainerManager containerManager = new MyContainerManager(); - final MockRM rm = - new MockRMWithAMS(confWithSecurityEnabled, containerManager); - rm.start(); - - final Configuration conf = rm.getConfig(); - final YarnRPC rpc = YarnRPC.create(conf); - ApplicationMasterProtocol rmClient = null; - - try { - MockNM nm1 = rm.registerNode("localhost:1234", 5120); - - RMApp app = rm.submitApp(1024); - - nm1.nodeHeartbeat(true); - - int waitCount = 0; - while (containerManager.amTokens == null && waitCount++ < 20) { - LOG.info("Waiting for AM Launch to happen.."); - Thread.sleep(1000); - } - Assert.assertNotNull(containerManager.amTokens); - - RMAppAttempt attempt = app.getCurrentAppAttempt(); - ApplicationAttemptId applicationAttemptId = attempt.getAppAttemptId(); - - // Create a client to the RM. - UserGroupInformation currentUser = - UserGroupInformation - .createRemoteUser(applicationAttemptId.toString()); - Credentials credentials = new Credentials(); - DataInputByteBuffer buf = new DataInputByteBuffer(); - containerManager.amTokens.rewind(); - buf.reset(containerManager.amTokens); - credentials.readTokenStorageStream(buf); - currentUser.addCredentials(credentials); - - rmClient = createRMClient(rm, conf, rpc, currentUser); - - RegisterApplicationMasterRequest request = - Records.newRecord(RegisterApplicationMasterRequest.class); - request.setApplicationAttemptId(applicationAttemptId); - rmClient.registerApplicationMaster(request); - - // One allocate call. - AllocateRequest allocateRequest = - Records.newRecord(AllocateRequest.class); - allocateRequest.setApplicationAttemptId(applicationAttemptId); - Assert.assertTrue( - rmClient.allocate(allocateRequest).getAMCommand() == null); - - // Simulate a master-key-roll-over - ApplicationTokenSecretManager appTokenSecretManager = - rm.getRMContext().getApplicationTokenSecretManager(); - SecretKey oldKey = appTokenSecretManager.getMasterKey(); - appTokenSecretManager.rollMasterKey(); - SecretKey newKey = appTokenSecretManager.getMasterKey(); - Assert.assertFalse("Master key should have changed!", - oldKey.equals(newKey)); - - // Another allocate call. Should continue to work. - rpc.stopProxy(rmClient, conf); // To avoid using cached client - rmClient = createRMClient(rm, conf, rpc, currentUser); - allocateRequest = Records.newRecord(AllocateRequest.class); - allocateRequest.setApplicationAttemptId(applicationAttemptId); - Assert.assertTrue( - rmClient.allocate(allocateRequest).getAMCommand() == null); - } finally { - rm.stop(); - if (rmClient != null) { - rpc.stopProxy(rmClient, conf); // To avoid using cached client - } - } - } - - private ApplicationMasterProtocol createRMClient(final MockRM rm, - final Configuration conf, final YarnRPC rpc, - UserGroupInformation currentUser) { - return currentUser.doAs(new PrivilegedAction() { - @Override - public ApplicationMasterProtocol run() { - return (ApplicationMasterProtocol) rpc.getProxy(ApplicationMasterProtocol.class, rm - .getApplicationMasterService().getBindAddress(), conf); - } - }); - } -} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java index 2706340..a4ccb43 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java @@ -67,13 +67,13 @@ import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.ipc.YarnRPC; -import org.apache.hadoop.yarn.security.ApplicationTokenIdentifier; +import org.apache.hadoop.yarn.security.AMRMTokenIdentifier; import org.apache.hadoop.yarn.security.ContainerTokenIdentifier; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState; -import org.apache.hadoop.yarn.server.resourcemanager.security.ApplicationTokenSecretManager; +import org.apache.hadoop.yarn.server.resourcemanager.security.AMRMTokenSecretManager; import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager; import org.apache.hadoop.yarn.server.utils.BuilderUtils; import org.apache.hadoop.yarn.util.Records; @@ -459,14 +459,14 @@ private ApplicationMasterProtocol submitAndRegisterApplication( final InetSocketAddress schedulerAddr = resourceManager.getApplicationMasterService().getBindAddress(); if (UserGroupInformation.isSecurityEnabled()) { - ApplicationTokenIdentifier appTokenIdentifier = new ApplicationTokenIdentifier( + AMRMTokenIdentifier appTokenIdentifier = new AMRMTokenIdentifier( appAttempt.getAppAttemptId()); - ApplicationTokenSecretManager appTokenSecretManager = - new ApplicationTokenSecretManager(conf); + AMRMTokenSecretManager appTokenSecretManager = + new AMRMTokenSecretManager(conf); appTokenSecretManager.setMasterKey(resourceManager - .getApplicationTokenSecretManager().getMasterKey()); - Token appToken = - new Token(appTokenIdentifier, + .getAMRMTokenSecretManager().getMasterKey()); + Token appToken = + new Token(appTokenIdentifier, appTokenSecretManager); SecurityUtil.setTokenService(appToken, schedulerAddr); currentUser.addToken(appToken);