diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/SecretManagerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/SecretManagerService.java new file mode 100644 index 0000000..0b455b7 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/SecretManagerService.java @@ -0,0 +1,197 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +package org.apache.hadoop.yarn.security; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Evolving; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.token.SecretManager; +import org.apache.hadoop.security.token.TokenIdentifier; +import org.apache.hadoop.service.AbstractService; +import org.apache.hadoop.service.LifecycleEvent; +import org.apache.hadoop.service.Service; +import org.apache.hadoop.service.ServiceStateChangeListener; +import org.apache.hadoop.yarn.security.ServiceHandler; + +import java.io.IOException; +import java.util.*; + +/** + * SecretManager wrapper class for YARN to treat ServiceManager as Service. + */ +@Public +@Evolving +public abstract class SecretManagerService extends + SecretManager implements Service { + private static Log LOG = LogFactory + .getLog(SecretManagerService.class); + private final SecretManagerServiceInternal secretManagerService; + + public SecretManagerService(String serviceName) { + secretManagerService = + new SecretManagerServiceInternal(serviceName); + } + + @Override + public void init(Configuration config) { + secretManagerService.init(config); + } + + @Override + public void start() { + secretManagerService.start(); + } + + @Override + public void stop() { + secretManagerService.stop(); + } + + @Override + public void close() throws IOException { + secretManagerService.close(); + } + + @Override + public void registerServiceListener(ServiceStateChangeListener listener) { + secretManagerService.registerServiceListener(listener); + } + + @Override + public void unregisterServiceListener(ServiceStateChangeListener listener) { + secretManagerService.unregisterServiceListener(listener); + } + + @Override + public String getName() { + return secretManagerService.getName(); + } + + @Override + public Configuration getConfig() { + return secretManagerService.getConfig(); + } + + @Override + public STATE getServiceState() { + return secretManagerService.getServiceState(); + } + + @Override + public long getStartTime() { + return secretManagerService.getStartTime(); + } + + @Override + public boolean isInState(STATE state) { + return secretManagerService.isInState(state); + } + + @Override + public Throwable getFailureCause() { + return secretManagerService.getFailureCause(); + } + + @Override + public STATE getFailureState() { + return secretManagerService.getFailureState(); + } + + @Override + public boolean waitForServiceToStop(long timeout) { + return secretManagerService.waitForServiceToStop(timeout); + } + + @Override + public List getLifecycleHistory() { + return secretManagerService.getLifecycleHistory(); + } + + @Override + public Map getBlockers() { + return secretManagerService.getBlockers(); + } + + /* ===================================================================== */ + /* Override Points */ + /* ===================================================================== */ + + /** + * All initialization code needed by a service. + * + * This method will only ever be called once during the lifecycle of + * a specific service instance. + * + * Implementations do not need to be synchronized as the logic + * in {@link #init(Configuration)} prevents re-entrancy. + * + * The base implementation checks to see if the subclass has created + * a new configuration instance, and if so, updates the base class value + * @param conf configuration + * @throws Exception on a failure -these will be caught, + * possibly wrapped, and wil; trigger a service stop + */ + protected void serviceInit(Configuration conf) throws Exception { + if (conf != getConfig()) { + LOG.debug("Config has been overridden during init"); + secretManagerService.setConfig(conf); + } + } + + /** + * Actions called during the INITED to STARTED transition. + * + * This method will only ever be called once during the lifecycle of + * a specific service instance. + * + * Implementations do not need to be synchronized as the logic + * in {@link #start()} prevents re-entrancy. + * + * @throws Exception if needed -these will be caught, + * wrapped, and trigger a service stop + */ + protected void serviceStart() throws Exception { + + } + + /** + * Actions called during the transition to the STOPPED state. + * + * This method will only ever be called once during the lifecycle of + * a specific service instance. + * + * Implementations do not need to be synchronized as the logic + * in {@link #stop()} prevents re-entrancy. + * + * Implementations MUST write this to be robust against failures, including + * checks for null references -and for the first failure to not stop other + * attempts to shut down parts of the service. + * + * @throws Exception if needed -these will be caught and logged. + */ + protected void serviceStop() throws Exception { + + } + + public void registerServiceHandler(ServiceHandler handler) { + secretManagerService.registerServiceHandler(handler); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/SecretManagerServiceInternal.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/SecretManagerServiceInternal.java new file mode 100644 index 0000000..556d0e4 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/SecretManagerServiceInternal.java @@ -0,0 +1,89 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +package org.apache.hadoop.yarn.security; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Evolving; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.token.SecretManager; +import org.apache.hadoop.security.token.TokenIdentifier; +import org.apache.hadoop.service.AbstractService; +import org.apache.hadoop.service.LifecycleEvent; +import org.apache.hadoop.service.Service; +import org.apache.hadoop.service.ServiceStateChangeListener; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +/** + * SecretManager wrapper class for YARN to treat ServiceManager as Service. + */ +@Public +@Evolving +public class SecretManagerServiceInternal extends AbstractService { + ArrayList serviceHandlers; + /** + * Construct the service. + * + * @param name service name + */ + public SecretManagerServiceInternal(String name) { + super(name); + serviceHandlers = new ArrayList(); + } + + @Override + protected void serviceInit(Configuration conf) throws Exception { + for (ServiceHandler handler :serviceHandlers){ + handler.serviceInit(conf); + } + } + @Override + protected void serviceStart() throws Exception { + for (ServiceHandler handler :serviceHandlers){ + handler.serviceStart(); + } + } + @Override + protected void serviceStop() throws Exception { + // serviceStop callbacks handlers in reverse order of + // "serviceStart" method. + Collections.reverse(serviceHandlers); + try { + for (ServiceHandler handler : serviceHandlers){ + handler.serviceStop(); + } + } finally { + Collections.reverse(serviceHandlers); + } + } + + public void setConfig(Configuration conf) { + super.setConfig(conf); + } + + public void registerServiceHandler(ServiceHandler handler) { + serviceHandlers.add(handler); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ServiceHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ServiceHandler.java new file mode 100644 index 0000000..1d6b34f --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ServiceHandler.java @@ -0,0 +1,36 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +package org.apache.hadoop.yarn.security; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Evolving; +import org.apache.hadoop.conf.Configuration; + +/** + * ServiceHandler interface is callback interface for SecretManagerService. + * These callbacks are identical to AbstractService class's methods and + * callbacked from same-name methods. + */ +@Public +@Evolving +public interface ServiceHandler { + public void serviceInit(Configuration conf) throws Exception; + public void serviceStart() throws Exception; + public void serviceStop() throws Exception; +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/BaseClientToAMTokenSecretManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/BaseClientToAMTokenSecretManager.java index c455aac..f8ca90c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/BaseClientToAMTokenSecretManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/BaseClientToAMTokenSecretManager.java @@ -25,6 +25,7 @@ import org.apache.hadoop.classification.InterfaceStability.Evolving; import org.apache.hadoop.security.token.SecretManager; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.security.SecretManagerService; /** * A base {@link SecretManager} for AMs to extend and validate Client-RM tokens @@ -36,7 +37,11 @@ @Public @Evolving public abstract class BaseClientToAMTokenSecretManager extends - SecretManager { + SecretManagerService { + + public BaseClientToAMTokenSecretManager(String serviceName) { + super(serviceName); + } @Private public abstract SecretKey getMasterKey( diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/ClientToAMTokenSecretManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/ClientToAMTokenSecretManager.java index 541f7a8..afc5048 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/ClientToAMTokenSecretManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/ClientToAMTokenSecretManager.java @@ -42,7 +42,7 @@ public ClientToAMTokenSecretManager( ApplicationAttemptId applicationAttemptID, byte[] key) { - super(); + super(ClientToAMTokenSecretManager.class.getName()); if (key != null) { this.masterKey = SecretManager.createSecretKey(key); } else { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/DelegationTokenSecretManagerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/DelegationTokenSecretManagerService.java new file mode 100644 index 0000000..0d4dbac --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/DelegationTokenSecretManagerService.java @@ -0,0 +1,139 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +package org.apache.hadoop.yarn.security.client; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Evolving; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier; +import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager; +import org.apache.hadoop.service.LifecycleEvent; +import org.apache.hadoop.service.Service; +import org.apache.hadoop.service.ServiceStateChangeListener; +import org.apache.hadoop.yarn.security.SecretManagerServiceInternal; +import org.apache.hadoop.yarn.security.ServiceHandler; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +/** + * AbstractDelegationSecretManager wrapper class for YARN to treat ServiceManager as Service. + */ +@Public +@Evolving +public abstract +class DelegationTokenSecretManagerService + extends AbstractDelegationTokenSecretManager implements Service { + private final SecretManagerServiceInternal secretManagerService; + + public DelegationTokenSecretManagerService(String serviceName, + long delegationKeyUpdateInterval, + long delegationTokenMaxLifetime, + long delegationTokenRenewInterval, + long delegationTokenRemoverScanInterval) { + super(delegationKeyUpdateInterval, delegationTokenMaxLifetime, + delegationTokenRenewInterval, delegationTokenRemoverScanInterval); + secretManagerService = new SecretManagerServiceInternal(serviceName); + } + + @Override + public void init(Configuration config) { + secretManagerService.init(config); + } + + @Override + public void start() { + secretManagerService.start(); + } + + @Override + public void stop() { + secretManagerService.stop(); + } + + @Override + public void close() throws IOException { + secretManagerService.close(); + } + + @Override + public void registerServiceListener(ServiceStateChangeListener listener) { + secretManagerService.registerServiceListener(listener); + } + + @Override + public void unregisterServiceListener(ServiceStateChangeListener listener) { + secretManagerService.unregisterServiceListener(listener); + } + + @Override + public String getName() { + return secretManagerService.getName(); + } + + @Override + public Configuration getConfig() { + return secretManagerService.getConfig(); + } + + @Override + public STATE getServiceState() { + return secretManagerService.getServiceState(); + } + + @Override + public long getStartTime() { + return secretManagerService.getStartTime(); + } + + @Override + public boolean isInState(STATE state) { + return secretManagerService.isInState(state); + } + + @Override + public Throwable getFailureCause() { + return secretManagerService.getFailureCause(); + } + + @Override + public STATE getFailureState() { + return secretManagerService.getFailureState(); + } + + @Override + public boolean waitForServiceToStop(long timeout) { + return secretManagerService.waitForServiceToStop(timeout); + } + + @Override + public List getLifecycleHistory() { + return secretManagerService.getLifecycleHistory(); + } + + @Override + public Map getBlockers() { + return secretManagerService.getBlockers(); + } + + public void registerServiceHandler(ServiceHandler handler) { + secretManagerService.registerServiceHandler(handler); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/BaseContainerTokenSecretManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/BaseContainerTokenSecretManager.java index ccfe8f5..cdc2d82 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/BaseContainerTokenSecretManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/BaseContainerTokenSecretManager.java @@ -30,6 +30,8 @@ import org.apache.hadoop.security.token.SecretManager; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.security.ContainerTokenIdentifier; +import org.apache.hadoop.yarn.security.SecretManagerService; +import org.apache.hadoop.yarn.security.ServiceHandler; import org.apache.hadoop.yarn.server.api.records.MasterKey; /** @@ -38,7 +40,7 @@ * */ public class BaseContainerTokenSecretManager extends - SecretManager { + SecretManagerService { private static Log LOG = LogFactory .getLog(BaseContainerTokenSecretManager.class); @@ -56,12 +58,31 @@ */ protected MasterKeyData currentMasterKey; - protected final long containerTokenExpiryInterval; + protected long containerTokenExpiryInterval; - public BaseContainerTokenSecretManager(Configuration conf) { - this.containerTokenExpiryInterval = + private class BaseContainerTokenServiceHandler implements ServiceHandler { + + @Override + public void serviceInit(Configuration conf) throws Exception { + containerTokenExpiryInterval = conf.getInt(YarnConfiguration.RM_CONTAINER_ALLOC_EXPIRY_INTERVAL_MS, YarnConfiguration.DEFAULT_RM_CONTAINER_ALLOC_EXPIRY_INTERVAL_MS); + } + + @Override + public void serviceStart() throws Exception { + //To change body of implemented methods use File | Settings | File Templates. + } + + @Override + public void serviceStop() throws Exception { + //To change body of implemented methods use File | Settings | File Templates. + } + } + + public BaseContainerTokenSecretManager(String serviceName) { + super(serviceName); + registerServiceHandler(new BaseContainerTokenServiceHandler()); } // Need lock as we increment serialNo etc. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/BaseNMTokenSecretManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/BaseNMTokenSecretManager.java index 01da1af..f6c2cd4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/BaseNMTokenSecretManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/BaseNMTokenSecretManager.java @@ -29,15 +29,15 @@ import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.security.SecurityUtil; -import org.apache.hadoop.security.token.SecretManager; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.Token; import org.apache.hadoop.yarn.security.NMTokenIdentifier; +import org.apache.hadoop.yarn.security.SecretManagerService; import org.apache.hadoop.yarn.server.api.records.MasterKey; public class BaseNMTokenSecretManager extends - SecretManager { + SecretManagerService { private static Log LOG = LogFactory .getLog(BaseNMTokenSecretManager.class); @@ -49,7 +49,11 @@ protected final Lock writeLock = readWriteLock.writeLock(); protected MasterKeyData currentMasterKey; - + + public BaseNMTokenSecretManager(String serviceName) { + super(serviceName); + } + protected MasterKeyData createNewMasterKey() { this.writeLock.lock(); try { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java index a169c12..45f3ecd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java @@ -130,7 +130,8 @@ protected void serviceInit(Configuration conf) throws Exception { conf.setBoolean(Dispatcher.DISPATCHER_EXIT_ON_ERROR_KEY, true); NMContainerTokenSecretManager containerTokenSecretManager = - new NMContainerTokenSecretManager(conf); + new NMContainerTokenSecretManager(); + addService(containerTokenSecretManager); NMTokenSecretManagerInNM nmTokenSecretManager = new NMTokenSecretManagerInNM(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/security/LocalizerTokenSecretManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/security/LocalizerTokenSecretManager.java index 5eff43b..b8258d5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/security/LocalizerTokenSecretManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/security/LocalizerTokenSecretManager.java @@ -20,14 +20,15 @@ import javax.crypto.SecretKey; -import org.apache.hadoop.security.token.SecretManager; +import org.apache.hadoop.yarn.security.SecretManagerService; public class LocalizerTokenSecretManager extends - SecretManager { + SecretManagerService { private final SecretKey secretKey; public LocalizerTokenSecretManager() { + super(LocalizerTokenSecretManager.class.getName()); this.secretKey = generateSecret(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/security/NMContainerTokenSecretManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/security/NMContainerTokenSecretManager.java index 8860a95..91ef26a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/security/NMContainerTokenSecretManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/security/NMContainerTokenSecretManager.java @@ -32,6 +32,7 @@ import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.security.ContainerTokenIdentifier; +import org.apache.hadoop.yarn.security.ServiceHandler; import org.apache.hadoop.yarn.server.api.records.MasterKey; import org.apache.hadoop.yarn.server.security.BaseContainerTokenSecretManager; import org.apache.hadoop.yarn.server.security.MasterKeyData; @@ -48,15 +49,15 @@ .getLog(NMContainerTokenSecretManager.class); private MasterKeyData previousMasterKey; - private final TreeMap> recentlyStartedContainerTracker; + private TreeMap> recentlyStartedContainerTracker; private String nodeHostAddr; - - public NMContainerTokenSecretManager(Configuration conf) { - super(conf); + + public NMContainerTokenSecretManager() { + super(NMContainerTokenSecretManager.class.getName()); recentlyStartedContainerTracker = - new TreeMap>(); + new TreeMap>(); } /** diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/security/NMTokenSecretManagerInNM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/security/NMTokenSecretManagerInNM.java index 9569fdc..1521320 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/security/NMTokenSecretManagerInNM.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/security/NMTokenSecretManagerInNM.java @@ -49,6 +49,7 @@ public NMTokenSecretManagerInNM() { + super(NMTokenSecretManagerInNM.class.getName()); this.oldMasterKeys = new HashMap(); appToAppAttemptMap = diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java index 9cd8f95..59b69d4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java @@ -78,7 +78,7 @@ public void testSuccessfulContainerLaunch() throws InterruptedException, YarnConfiguration conf = new YarnConfiguration(); - Context context = new NMContext(new NMContainerTokenSecretManager(conf), + Context context = new NMContext(new NMContainerTokenSecretManager(), new NMTokenSecretManagerInNM(), null, null) { @Override public int getHttpPort() { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java index 4f23427..a633592 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java @@ -102,8 +102,8 @@ public BaseContainerManagerTest() throws UnsupportedFileSystemException { protected static final int HTTP_PORT = 5412; protected Configuration conf = new YarnConfiguration(); - protected Context context = new NMContext(new NMContainerTokenSecretManager( - conf), new NMTokenSecretManagerInNM(), null, new ApplicationACLsManager(conf)) { + protected Context context = new NMContext(new NMContainerTokenSecretManager(), + new NMTokenSecretManagerInNM(), null, new ApplicationACLsManager(conf)) { public int getHttpPort() { return HTTP_PORT; }; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/TestApplication.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/TestApplication.java index 356029e..8307c07 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/TestApplication.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/TestApplication.java @@ -515,7 +515,7 @@ public boolean matches(Object argument) { context = mock(Context.class); when(context.getContainerTokenSecretManager()).thenReturn( - new NMContainerTokenSecretManager(conf)); + new NMContainerTokenSecretManager()); when(context.getApplicationACLsManager()).thenReturn( new ApplicationACLsManager(conf)); when(context.getNMTokenSecretManager()).thenReturn(nmTokenSecretMgr); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java index e46c2bf..f55cd49 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java @@ -198,14 +198,12 @@ protected void setRMStateStore(RMStateStore rmStore) { ((RMContextImpl) rmContext).setStateStore(rmStore); } - protected RMContainerTokenSecretManager createContainerTokenSecretManager( - Configuration conf) { - return new RMContainerTokenSecretManager(conf); + protected RMContainerTokenSecretManager createContainerTokenSecretManager() { + return new RMContainerTokenSecretManager(); } - protected NMTokenSecretManagerInRM createNMTokenSecretManager( - Configuration conf) { - return new NMTokenSecretManagerInRM(conf); + protected NMTokenSecretManagerInRM createNMTokenSecretManager() { + return new NMTokenSecretManagerInRM(); } protected EventHandler createSchedulerEventDispatcher() { @@ -216,9 +214,8 @@ protected Dispatcher createDispatcher() { return new AsyncDispatcher(); } - protected AMRMTokenSecretManager createAMRMTokenSecretManager( - Configuration conf) { - return new AMRMTokenSecretManager(conf); + protected AMRMTokenSecretManager createAMRMTokenSecretManager() { + return new AMRMTokenSecretManager(); } protected ResourceScheduler createScheduler() { @@ -305,7 +302,8 @@ protected void serviceInit(Configuration configuration) throws Exception { rmDispatcher = createDispatcher(); addIfService(rmDispatcher); - amRmTokenSecretManager = createAMRMTokenSecretManager(conf); + amRmTokenSecretManager = createAMRMTokenSecretManager(); + addService(amRmTokenSecretManager); containerAllocationExpirer = new ContainerAllocationExpirer(rmDispatcher); addService(containerAllocationExpirer); @@ -316,8 +314,10 @@ protected void serviceInit(Configuration configuration) throws Exception { AMLivelinessMonitor amFinishingMonitor = createAMLivelinessMonitor(); addService(amFinishingMonitor); - containerTokenSecretManager = createContainerTokenSecretManager(conf); - nmTokenSecretManager = createNMTokenSecretManager(conf); + containerTokenSecretManager = createContainerTokenSecretManager(); + addService(containerTokenSecretManager); + nmTokenSecretManager = createNMTokenSecretManager(); + addService(nmTokenSecretManager); boolean isRecoveryEnabled = conf.getBoolean( YarnConfiguration.RECOVERY_ENABLED, @@ -404,6 +404,7 @@ protected void serviceInit(Configuration configuration) throws Exception { // Register event handler for RMAppManagerEvents rmDispatcher.register(RMAppManagerEventType.class, rmAppManager); rmDTSecretManager = createRMDelegationTokenSecretManager(rmContext); + addService(rmDTSecretManager); rmContext.setRMDelegationTokenSecretManager(rmDTSecretManager); clientRM = createClientRMService(); rmContext.setClientRMService(clientRM); @@ -429,10 +430,6 @@ protected void serviceInit(Configuration configuration) throws Exception { @Override protected void serviceStart() throws Exception { - amRmTokenSecretManager.start(); - containerTokenSecretManager.start(); - nmTokenSecretManager.start(); - RMStateStore rmStore = rmContext.getStateStore(); // The state store needs to start irrespective of recoveryEnabled as apps // need events to move to further states. @@ -451,11 +448,6 @@ protected void serviceStart() throws Exception { } startWepApp(); - try { - rmDTSecretManager.startThreads(); - } catch(IOException ie) { - throw new YarnRuntimeException("Failed to start secret manager threads", ie); - } if (getConfig().getBoolean(YarnConfiguration.IS_MINI_YARN_CLUSTER, false)) { int port = webApp.port(); @@ -470,20 +462,6 @@ protected void serviceStop() throws Exception { if (webApp != null) { webApp.stop(); } - if (rmDTSecretManager != null) { - rmDTSecretManager.stopThreads(); - } - - if (amRmTokenSecretManager != null) { - amRmTokenSecretManager.stop(); - } - if (containerTokenSecretManager != null) { - containerTokenSecretManager.stop(); - } - if(nmTokenSecretManager != null) { - nmTokenSecretManager.stop(); - } - DefaultMetricsSystem.shutdown(); if (rmContext != null) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/AMRMTokenSecretManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/AMRMTokenSecretManager.java index 5d21ec0..b1e8c5a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/AMRMTokenSecretManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/AMRMTokenSecretManager.java @@ -30,11 +30,12 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.security.token.SecretManager; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.security.AMRMTokenIdentifier; +import org.apache.hadoop.yarn.security.SecretManagerService; +import org.apache.hadoop.yarn.security.ServiceHandler; /** * AMRM-tokens are per ApplicationAttempt. If users redistribute their @@ -44,37 +45,53 @@ * so no need to remember master-keys even after rolling them. */ public class AMRMTokenSecretManager extends - SecretManager { + SecretManagerService { private static final Log LOG = LogFactory .getLog(AMRMTokenSecretManager.class); private SecretKey masterKey; - private final Timer timer; - private final long rollingInterval; private final Map passwords = new HashMap(); - /** - * Create an {@link AMRMTokenSecretManager} - */ - public AMRMTokenSecretManager(Configuration conf) { - rollMasterKey(); - this.timer = new Timer(); - this.rollingInterval = + private class AMRMTokenServiceHandler implements ServiceHandler { + private Timer timer; + private long rollingInterval; + + @Override + public void serviceInit(Configuration conf) throws Exception { + this.timer = new Timer(); + this.rollingInterval = conf .getLong( YarnConfiguration.RM_AMRM_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS, YarnConfiguration.DEFAULT_RM_AMRM_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS) * 1000; + } + + @Override + public void serviceStart() throws Exception { + this.timer.scheduleAtFixedRate(new MasterKeyRoller(), 0, rollingInterval); + } + + @Override + public void serviceStop() throws Exception { + this.timer.cancel(); + } } - public void start() { - this.timer.scheduleAtFixedRate(new MasterKeyRoller(), 0, rollingInterval); + /** + * Create an {@link AMRMTokenSecretManager} + */ + public AMRMTokenSecretManager() { + super(AMRMTokenSecretManager.class.getName()); + registerServiceHandler(new AMRMTokenServiceHandler()); + rollMasterKey(); } - public void stop() { - this.timer.cancel(); + @Override + protected void serviceInit(Configuration conf) throws Exception { + } public synchronized void applicationMasterFinished( diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/ClientToAMTokenSecretManagerInRM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/ClientToAMTokenSecretManagerInRM.java index 4fbe2ce..996d3a6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/ClientToAMTokenSecretManagerInRM.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/ClientToAMTokenSecretManagerInRM.java @@ -33,6 +33,10 @@ private Map masterKeys = new HashMap(); + public ClientToAMTokenSecretManagerInRM() { + super(ClientToAMTokenSecretManagerInRM.class.getName()); + } + public synchronized SecretKey createMasterKey( ApplicationAttemptId applicationAttemptID) { return generateSecret(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/NMTokenSecretManagerInRM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/NMTokenSecretManagerInRM.java index ab31eaf..f150a98 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/NMTokenSecretManagerInRM.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/NMTokenSecretManagerInRM.java @@ -36,6 +36,7 @@ import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.Token; import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.security.ServiceHandler; import org.apache.hadoop.yarn.server.api.records.MasterKey; import org.apache.hadoop.yarn.server.security.BaseNMTokenSecretManager; import org.apache.hadoop.yarn.server.security.MasterKeyData; @@ -49,42 +50,60 @@ .getLog(NMTokenSecretManagerInRM.class); private MasterKeyData nextMasterKey; - private Configuration conf; - private final Timer timer; - private final long rollingInterval; - private final long activationDelay; + private Timer timer; + private long rollingInterval; + private long activationDelay; private final ConcurrentHashMap> appAttemptToNodeKeyMap; - - public NMTokenSecretManagerInRM(Configuration conf) { - this.conf = conf; - timer = new Timer(); - rollingInterval = this.conf.getLong( + + private class NMTokenSecretServiceHandler implements ServiceHandler { + + @Override + public void serviceInit(Configuration conf) throws Exception { + timer = new Timer(); + rollingInterval = conf.getLong( YarnConfiguration.RM_NMTOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS, YarnConfiguration.DEFAULT_RM_NMTOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS) * 1000; - // Add an activation delay. This is to address the following race: RM may - // roll over master-key, scheduling may happen at some point of time, an - // NMToken created with a password generated off new master key, but NM - // might not have come again to RM to update the shared secret: so AM has a - // valid password generated off new secret but NM doesn't know about the - // secret yet. - // Adding delay = 1.5 * expiry interval makes sure that all active NMs get - // the updated shared-key. - this.activationDelay = + // Add an activation delay. This is to address the following race: RM may + // roll over master-key, scheduling may happen at some point of time, an + // NMToken created with a password generated off new master key, but NM + // might not have come again to RM to update the shared secret: so AM has a + // valid password generated off new secret but NM doesn't know about the + // secret yet. + // Adding delay = 1.5 * expiry interval makes sure that all active NMs get + // the updated shared-key. + activationDelay = (long) (conf.getLong(YarnConfiguration.RM_NM_EXPIRY_INTERVAL_MS, - YarnConfiguration.DEFAULT_RM_NM_EXPIRY_INTERVAL_MS) * 1.5); - LOG.info("NMTokenKeyRollingInterval: " + this.rollingInterval - + "ms and NMTokenKeyActivationDelay: " + this.activationDelay + YarnConfiguration.DEFAULT_RM_NM_EXPIRY_INTERVAL_MS) * 1.5); + LOG.info("NMTokenKeyRollingInterval: " + rollingInterval + + "ms and NMTokenKeyActivationDelay: " + activationDelay + "ms"); - if (rollingInterval <= activationDelay * 2) { - throw new IllegalArgumentException( + if (rollingInterval <= activationDelay * 2) { + throw new IllegalArgumentException( YarnConfiguration.RM_NMTOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS - + " should be more than 2 X " - + YarnConfiguration.RM_NM_EXPIRY_INTERVAL_MS); + + " should be more than 2 X " + + YarnConfiguration.RM_NM_EXPIRY_INTERVAL_MS); + } + } + + @Override + public void serviceStart() throws Exception { + rollMasterKey(); + timer.scheduleAtFixedRate(new MasterKeyRoller(), rollingInterval, + rollingInterval); } + + @Override + public void serviceStop() throws Exception { + timer.cancel(); + } + } + + public NMTokenSecretManagerInRM() { + super(NMTokenSecretManagerInRM.class.getName()); appAttemptToNodeKeyMap = - new ConcurrentHashMap>(); + new ConcurrentHashMap>(); } /** @@ -151,16 +170,6 @@ private void clearApplicationNMTokenKeys() { } } - public void start() { - rollMasterKey(); - this.timer.scheduleAtFixedRate(new MasterKeyRoller(), rollingInterval, - rollingInterval); - } - - public void stop() { - this.timer.cancel(); - } - private class MasterKeyRoller extends TimerTask { @Override public void run() { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/RMContainerTokenSecretManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/RMContainerTokenSecretManager.java index c62f2ee..b987902 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/RMContainerTokenSecretManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/RMContainerTokenSecretManager.java @@ -31,6 +31,7 @@ import org.apache.hadoop.yarn.api.records.Token; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.security.ContainerTokenIdentifier; +import org.apache.hadoop.yarn.security.ServiceHandler; import org.apache.hadoop.yarn.server.api.records.MasterKey; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.security.BaseContainerTokenSecretManager; @@ -50,47 +51,56 @@ private MasterKeyData nextMasterKey; - private final Timer timer; - private final long rollingInterval; - private final long activationDelay; - - public RMContainerTokenSecretManager(Configuration conf) { - super(conf); - - this.timer = new Timer(); - this.rollingInterval = conf.getLong( - YarnConfiguration.RM_CONTAINER_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS, - YarnConfiguration.DEFAULT_RM_CONTAINER_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS) * 1000; - // Add an activation delay. This is to address the following race: RM may - // roll over master-key, scheduling may happen at some point of time, a - // container created with a password generated off new master key, but NM - // might not have come again to RM to update the shared secret: so AM has a - // valid password generated off new secret but NM doesn't know about the - // secret yet. - // Adding delay = 1.5 * expiry interval makes sure that all active NMs get - // the updated shared-key. - this.activationDelay = + private Timer timer; + private long rollingInterval; + private long activationDelay; + + class RMContainerTokenServiceHandler implements ServiceHandler { + + @Override + public void serviceInit(Configuration conf) throws Exception { + timer = new Timer(); + rollingInterval = conf.getLong( + YarnConfiguration.RM_CONTAINER_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS, + YarnConfiguration.DEFAULT_RM_CONTAINER_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS) * 1000; + // Add an activation delay. This is to address the following race: RM may + // roll over master-key, scheduling may happen at some point of time, a + // container created with a password generated off new master key, but NM + // might not have come again to RM to update the shared secret: so AM has a + // valid password generated off new secret but NM doesn't know about the + // secret yet. + // Adding delay = 1.5 * expiry interval makes sure that all active NMs get + // the updated shared-key. + activationDelay = (long) (conf.getLong(YarnConfiguration.RM_NM_EXPIRY_INTERVAL_MS, - YarnConfiguration.DEFAULT_RM_NM_EXPIRY_INTERVAL_MS) * 1.5); - LOG.info("ContainerTokenKeyRollingInterval: " + this.rollingInterval - + "ms and ContainerTokenKeyActivationDelay: " + this.activationDelay + YarnConfiguration.DEFAULT_RM_NM_EXPIRY_INTERVAL_MS) * 1.5); + LOG.info("ContainerTokenKeyRollingInterval: " + rollingInterval + + "ms and ContainerTokenKeyActivationDelay: " + activationDelay + "ms"); - if (rollingInterval <= activationDelay * 2) { - throw new IllegalArgumentException( + if (rollingInterval <= activationDelay * 2) { + throw new IllegalArgumentException( YarnConfiguration.RM_CONTAINER_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS - + " should be more than 2 X " - + YarnConfiguration.RM_CONTAINER_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS); + + " should be more than 2 X " + + YarnConfiguration.RM_CONTAINER_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS); + } } - } - public void start() { - rollMasterKey(); - this.timer.scheduleAtFixedRate(new MasterKeyRoller(), rollingInterval, + @Override + public void serviceStart() throws Exception { + rollMasterKey(); + timer.scheduleAtFixedRate(new MasterKeyRoller(), rollingInterval, rollingInterval); + } + + @Override + public void serviceStop() throws Exception { + timer.cancel(); + } } - public void stop() { - this.timer.cancel(); + public RMContainerTokenSecretManager() { + super(RMContainerTokenSecretManager.class.getName()); + registerServiceHandler(new RMContainerTokenServiceHandler()); } /** diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/RMDelegationTokenSecretManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/RMDelegationTokenSecretManager.java index 23939de..a4dccfe 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/RMDelegationTokenSecretManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/RMDelegationTokenSecretManager.java @@ -29,9 +29,11 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability; -import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.token.delegation.DelegationKey; import org.apache.hadoop.util.ExitUtil; +import org.apache.hadoop.yarn.security.ServiceHandler; +import org.apache.hadoop.yarn.security.client.DelegationTokenSecretManagerService; import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.RMState; @@ -47,13 +49,31 @@ @InterfaceAudience.Private @InterfaceStability.Unstable public class RMDelegationTokenSecretManager extends - AbstractDelegationTokenSecretManager implements + DelegationTokenSecretManagerService implements Recoverable { private static final Log LOG = LogFactory .getLog(RMDelegationTokenSecretManager.class); protected final RMContext rmContext; + + private class RMDelegationTokenServiceHandler implements ServiceHandler { + + @Override + public void serviceInit(Configuration conf) throws Exception { + startThreads(); + } + + @Override + public void serviceStart() throws Exception { + } + + @Override + public void serviceStop() throws Exception { + stopThreads(); + } + } + /** * Create a secret manager * @param delegationKeyUpdateInterval the number of seconds for rolling new @@ -69,8 +89,10 @@ public RMDelegationTokenSecretManager(long delegationKeyUpdateInterval, long delegationTokenRenewInterval, long delegationTokenRemoverScanInterval, RMContext rmContext) { - super(delegationKeyUpdateInterval, delegationTokenMaxLifetime, + super(RMDelegationTokenSecretManager.class.getName(), + delegationKeyUpdateInterval, delegationTokenMaxLifetime, delegationTokenRenewInterval, delegationTokenRemoverScanInterval); + registerServiceHandler(new RMDelegationTokenServiceHandler()); this.rmContext = rmContext; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java index 9ec82c4..d1397f1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java @@ -111,14 +111,12 @@ public static void setupSecretManager() throws IOException { RMContext rmContext = mock(RMContext.class); when(rmContext.getStateStore()).thenReturn(new NullRMStateStore()); dtsm = new RMDelegationTokenSecretManager(60000, 60000, 60000, 60000, rmContext); - dtsm.startThreads(); + dtsm.init(new Configuration()); } @AfterClass public static void teardownSecretManager() { - if (dtsm != null) { - dtsm.stopThreads(); - } + dtsm.stop(); } @Test diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMTokens.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMTokens.java index d389c0e..1fb7605 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMTokens.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMTokens.java @@ -99,7 +99,8 @@ public void testDelegationToken() throws IOException, InterruptedException { RMDelegationTokenSecretManager rmDtSecretManager = createRMDelegationTokenSecretManager( initialInterval, maxLifetime, renewInterval); - rmDtSecretManager.startThreads(); + rmDtSecretManager.init(conf); + rmDtSecretManager.start(); LOG.info("Creating DelegationTokenSecretManager with initialInterval: " + initialInterval + ", maxLifetime: " + maxLifetime + ", renewInterval: " + renewInterval); @@ -224,7 +225,7 @@ public void testDelegationToken() throws IOException, InterruptedException { } finally { - rmDtSecretManager.stopThreads(); + rmDtSecretManager.stop(); // TODO PRECOMMIT Close proxies. if (clientRMWithDT != null) { RPC.stopProxy(clientRMWithDT); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestRMStateStore.java index d75fc7d..c1601b9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestRMStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestRMStateStore.java @@ -75,7 +75,7 @@ import org.junit.Test; -public class TestRMStateStore extends ClientBaseWithFixes{ +public class TestRMStateStore extends ClientBaseWithFixes { public static final Log LOG = LogFactory.getLog(TestRMStateStore.class); @@ -274,7 +274,7 @@ void testRMAppStateStore(RMStateStoreHelper stateStoreHelper) store.setRMDispatcher(dispatcher); AMRMTokenSecretManager appTokenMgr = - new AMRMTokenSecretManager(conf); + new AMRMTokenSecretManager(); ClientToAMTokenSecretManagerInRM clientToAMTokenMgr = new ClientToAMTokenSecretManagerInRM(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestNMExpiry.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestNMExpiry.java index a884552..ead6aeb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestNMExpiry.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestNMExpiry.java @@ -84,10 +84,11 @@ public void setUp() { NodesListManager nodesListManager = new NodesListManager(context); nodesListManager.init(conf); RMContainerTokenSecretManager containerTokenSecretManager = - new RMContainerTokenSecretManager(conf); + new RMContainerTokenSecretManager(); + containerTokenSecretManager.init(conf); containerTokenSecretManager.start(); NMTokenSecretManagerInRM nmTokenSecretManager = - new NMTokenSecretManagerInRM(conf); + new NMTokenSecretManagerInRM(); nmTokenSecretManager.start(); resourceTrackerService = new ResourceTrackerService(context, nodesListManager, nmLivelinessMonitor, containerTokenSecretManager, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestRMNMRPCResponseId.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestRMNMRPCResponseId.java index ddb7a90..71478f7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestRMNMRPCResponseId.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestRMNMRPCResponseId.java @@ -70,8 +70,8 @@ public void handle(Event event) { }); RMContext context = new RMContextImpl(dispatcher, null, null, null, null, - null, new RMContainerTokenSecretManager(conf), - new NMTokenSecretManagerInRM(conf), null); + null, new RMContainerTokenSecretManager(), + new NMTokenSecretManagerInRM(), null); dispatcher.register(RMNodeEventType.class, new ResourceManager.NodeEventDispatcher(context)); NodesListManager nodesListManager = new NodesListManager(context); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java index 2075921..b8d87b0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java @@ -190,9 +190,9 @@ public void setUp() throws Exception { this.rmContext = new RMContextImpl(rmDispatcher, store, containerAllocationExpirer, amLivelinessMonitor, amFinishingMonitor, - null, new AMRMTokenSecretManager(conf), - new RMContainerTokenSecretManager(conf), - new NMTokenSecretManagerInRM(conf), + null, new AMRMTokenSecretManager(), + new RMContainerTokenSecretManager(), + new NMTokenSecretManagerInRM(), new ClientToAMTokenSecretManagerInRM()); rmDispatcher.register(RMAppAttemptEventType.class, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java index 1f3c506..65b3508 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java @@ -119,7 +119,7 @@ private RMAppAttempt applicationAttempt; private Configuration conf = new Configuration(); - private AMRMTokenSecretManager amRMTokenManager = spy(new AMRMTokenSecretManager(conf)); + private AMRMTokenSecretManager amRMTokenManager = spy(new AMRMTokenSecretManager()); private ClientToAMTokenSecretManagerInRM clientToAMTokenManager = spy(new ClientToAMTokenSecretManagerInRM()); @@ -205,8 +205,8 @@ public void setUp() throws Exception { new RMContextImpl(rmDispatcher, containerAllocationExpirer, amLivelinessMonitor, amFinishingMonitor, null, amRMTokenManager, - new RMContainerTokenSecretManager(conf), - new NMTokenSecretManagerInRM(conf), + new RMContainerTokenSecretManager(), + new NMTokenSecretManagerInRM(), clientToAMTokenManager); RMStateStore store = mock(RMStateStore.class); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java index f343bd5..e0cae91 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java @@ -90,7 +90,7 @@ public void setUp() throws IOException { when(csContext.getResourceCalculator()). thenReturn(resourceCalculator); RMContainerTokenSecretManager containerTokenSecretManager = - new RMContainerTokenSecretManager(conf); + new RMContainerTokenSecretManager(); containerTokenSecretManager.rollMasterKey(); when(csContext.getContainerTokenSecretManager()).thenReturn( containerTokenSecretManager); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java index 627fae8..98efb04 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java @@ -343,8 +343,8 @@ public void testRefreshQueues() throws Exception { setupQueueConfiguration(conf); cs.setConf(new YarnConfiguration()); cs.reinitialize(conf, new RMContextImpl(null, null, null, null, null, - null, new RMContainerTokenSecretManager(conf), - new NMTokenSecretManagerInRM(conf), + null, new RMContainerTokenSecretManager(), + new NMTokenSecretManagerInRM(), new ClientToAMTokenSecretManagerInRM())); checkQueueCapacities(cs, A_CAPACITY, B_CAPACITY); @@ -442,8 +442,8 @@ public void testParseQueue() throws IOException { conf.setUserLimitFactor(CapacitySchedulerConfiguration.ROOT + ".a.a1.b1", 100.0f); cs.reinitialize(conf, new RMContextImpl(null, null, null, null, null, - null, new RMContainerTokenSecretManager(conf), - new NMTokenSecretManagerInRM(conf), + null, new RMContainerTokenSecretManager(), + new NMTokenSecretManagerInRM(), new ClientToAMTokenSecretManagerInRM())); } @@ -455,8 +455,8 @@ public void testReconnectedNode() throws Exception { CapacityScheduler cs = new CapacityScheduler(); cs.setConf(new YarnConfiguration()); cs.reinitialize(csConf, new RMContextImpl(null, null, null, null, - null, null, new RMContainerTokenSecretManager(csConf), - new NMTokenSecretManagerInRM(csConf), + null, null, new RMContainerTokenSecretManager(), + new NMTokenSecretManagerInRM(), new ClientToAMTokenSecretManagerInRM())); RMNode n1 = MockNodes.newNodeInfo(0, MockNodes.newResource(4 * GB), 1); @@ -482,8 +482,8 @@ public void testRefreshQueuesWithNewQueue() throws Exception { setupQueueConfiguration(conf); cs.setConf(new YarnConfiguration()); cs.reinitialize(conf, new RMContextImpl(null, null, null, null, null, - null, new RMContainerTokenSecretManager(conf), - new NMTokenSecretManagerInRM(conf), + null, new RMContainerTokenSecretManager(), + new NMTokenSecretManagerInRM(), new ClientToAMTokenSecretManagerInRM())); checkQueueCapacities(cs, A_CAPACITY, B_CAPACITY); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java index 4f4bf2f..efc77db 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java @@ -124,7 +124,7 @@ public void setUp() throws Exception { when(csContext.getResourceCalculator()). thenReturn(resourceCalculator); RMContainerTokenSecretManager containerTokenSecretManager = - new RMContainerTokenSecretManager(conf); + new RMContainerTokenSecretManager(); containerTokenSecretManager.rollMasterKey(); when(csContext.getContainerTokenSecretManager()).thenReturn( containerTokenSecretManager); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java index c86d6b3..4b48a47 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java @@ -45,8 +45,8 @@ public void testQueueParsing() throws Exception { CapacityScheduler capacityScheduler = new CapacityScheduler(); capacityScheduler.setConf(conf); capacityScheduler.reinitialize(conf, new RMContextImpl(null, null, - null, null, null, null, new RMContainerTokenSecretManager(conf), - new NMTokenSecretManagerInRM(conf), + null, null, null, null, new RMContainerTokenSecretManager(), + new NMTokenSecretManagerInRM(), new ClientToAMTokenSecretManagerInRM())); CSQueue a = capacityScheduler.getQueue("a"); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java index b974528..b289dfe 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java @@ -86,9 +86,9 @@ public EventHandler getEventHandler() { Configuration conf = new Configuration(); RMContext rmContext = new RMContextImpl(nullDispatcher, cae, null, null, null, - new AMRMTokenSecretManager(conf), - new RMContainerTokenSecretManager(conf), - new NMTokenSecretManagerInRM(conf), + new AMRMTokenSecretManager(), + new RMContainerTokenSecretManager(), + new NMTokenSecretManagerInRM(), new ClientToAMTokenSecretManagerInRM()); return rmContext; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java index ee302b9..7d85368 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java @@ -157,10 +157,10 @@ public void testNodeLocalAssignment() throws Exception { AsyncDispatcher dispatcher = new InlineDispatcher(); Configuration conf = new Configuration(); RMContainerTokenSecretManager containerTokenSecretManager = - new RMContainerTokenSecretManager(conf); + new RMContainerTokenSecretManager(); containerTokenSecretManager.rollMasterKey(); NMTokenSecretManagerInRM nmTokenSecretManager = - new NMTokenSecretManagerInRM(conf); + new NMTokenSecretManagerInRM(); nmTokenSecretManager.rollMasterKey(); RMContext rmContext = new RMContextImpl(dispatcher, null, null, null, null, null, containerTokenSecretManager, nmTokenSecretManager, null); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java index aa2d6c6..2ad240c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java @@ -202,8 +202,8 @@ public static CapacityScheduler mockCapacityScheduler() throws IOException { CapacityScheduler cs = new CapacityScheduler(); cs.setConf(new YarnConfiguration()); cs.reinitialize(conf, new RMContextImpl(null, null, null, null, null, - null, new RMContainerTokenSecretManager(conf), - new NMTokenSecretManagerInRM(conf), + null, new RMContainerTokenSecretManager(), + new NMTokenSecretManagerInRM(), new ClientToAMTokenSecretManagerInRM())); return cs; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java index 0a62f36..ffa9fc4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java @@ -188,7 +188,8 @@ private void testNMTokens(Configuration conf) throws Exception { StringBuilder sb; // testInvalidNMToken ... creating NMToken using different secret manager. - NMTokenSecretManagerInRM tempManager = new NMTokenSecretManagerInRM(conf); + NMTokenSecretManagerInRM tempManager = new NMTokenSecretManagerInRM(); + tempManager.init(conf); tempManager.rollMasterKey(); do { tempManager.rollMasterKey(); @@ -545,7 +546,8 @@ private void testContainerToken(Configuration conf) throws IOException, yarnCluster.getResourceManager().getRMContainerTokenSecretManager(); RMContainerTokenSecretManager tamperedContainerTokenSecretManager = - new RMContainerTokenSecretManager(conf); + new RMContainerTokenSecretManager(); + tamperedContainerTokenSecretManager.init(conf); tamperedContainerTokenSecretManager.rollMasterKey(); do { tamperedContainerTokenSecretManager.rollMasterKey();